Move query-based specimen import to Specimen module

specimen/src/org/labkey/specimen/SpecimenModule.java

@@ -57,6 +57,7 @@
 import org.labkey.api.study.importer.SimpleStudyImporterRegistry;
 import org.labkey.api.study.writer.SimpleStudyWriterRegistry;
 import org.labkey.api.usageMetrics.UsageMetricsService;
+import org.labkey.api.util.SystemMaintenance;
 import org.labkey.api.util.emailTemplate.EmailTemplateService;
 import org.labkey.api.util.logging.LogHelper;
 import org.labkey.api.view.ActionURL;
@@ -66,6 +67,8 @@
 import org.labkey.specimen.actions.SpecimenApiController;
 import org.labkey.specimen.actions.SpecimenController;
 import org.labkey.specimen.importer.AbstractSpecimenTask;
+import org.labkey.specimen.importer.QueryBasedSpecimenImportUploadTask;
+import org.labkey.specimen.importer.QueryBasedSpecimenTransform;
 import org.labkey.specimen.importer.RequestabilityManager;
 import org.labkey.specimen.importer.SpecimenImporter;
 import org.labkey.specimen.importer.SpecimenSchemaImporter;
@@ -232,7 +235,9 @@ public void addSpecimenPivotTableNames(Set<String> names)
                 return null;
             }
         });
-     }
+
+        SystemMaintenance.addTask(new QueryBasedSpecimenImportUploadTask());
+    }
 
     @Override
     protected void startupAfterSpringConfig(ModuleContext moduleContext)
@@ -254,6 +259,10 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext)
             new SpecimenSettingsImporter()
         ));
 
+        SpecimenService specimenService = SpecimenService.get();
+        if (null != specimenService)
+            specimenService.registerSpecimenTransform(new QueryBasedSpecimenTransform());
+
         UsageMetricsService svc = UsageMetricsService.get();
         if (null != svc)
         {

specimen/src/org/labkey/specimen/actions/SpecimenController.java

@@ -64,8 +64,10 @@
 import org.labkey.api.module.FolderType;
 import org.labkey.api.module.Module;
 import org.labkey.api.pipeline.PipeRoot;
+import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineService;
 import org.labkey.api.pipeline.PipelineStatusUrls;
+import org.labkey.api.pipeline.PipelineValidationException;
 import org.labkey.api.pipeline.browse.PipelinePathForm;
 import org.labkey.api.query.AbstractQueryImportAction;
 import org.labkey.api.query.BatchValidationException;
@@ -152,6 +154,7 @@
 import org.labkey.specimen.SpecimenRequestException;
 import org.labkey.specimen.SpecimenRequestManager;
 import org.labkey.specimen.SpecimenRequestStatus;
+import org.labkey.specimen.importer.QueryBasedSpecimenTransform;
 import org.labkey.specimen.importer.RequestabilityManager;
 import org.labkey.specimen.importer.SimpleSpecimenImporter;
 import org.labkey.specimen.model.ExtendedSpecimenRequestView;
@@ -195,7 +198,6 @@
 
 import java.io.BufferedOutputStream;
 import java.io.ByteArrayOutputStream;
-import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
@@ -5759,4 +5761,143 @@ public void addNavTrail(NavTree root)
             root.addChild("Insert " + _form.getQueryName());
         }
     }
+
+    public static class ConfigForm
+    {
+        private String _schemaName;
+        private String _queryName;
+        private String _viewName;
+        private String _enabled;
+        private int _userId;
+
+        public int getUserId()
+        {
+            return _userId;
+        }
+
+        public void setUserId(int userId)
+        {
+            _userId = userId;
+        }
+
+        public String getSchemaName()
+        {
+            return _schemaName;
+        }
+
+        public void setSchemaName(String schemaName)
+        {
+            _schemaName = schemaName;
+        }
+
+        public String getQueryName()
+        {
+            return _queryName;
+        }
+
+        public void setQueryName(String queryName)
+        {
+            _queryName = queryName;
+        }
+
+        public String getViewName()
+        {
+            return _viewName;
+        }
+
+        public void setViewName(String viewName)
+        {
+            _viewName = viewName;
+        }
+
+        public String getEnabled()
+        {
+            return _enabled;
+        }
+
+        public void setEnabled(String enabled)
+        {
+            _enabled = enabled;
+        }
+
+        public Map<String, String> getOptions()
+        {
+            Map<String, String> valueMap = new HashMap<>();
+            valueMap.put("schemaName", _schemaName);
+            valueMap.put("queryName", _queryName);
+            valueMap.put("viewName", _viewName);
+            valueMap.put("enabled", _enabled);
+            valueMap.put("userId", String.valueOf(_userId));
+            return valueMap;
+        }
+    }
+
+    @RequiresPermission(AdminPermission.class)
+    public static class ConfigureQueryImportAction extends FormViewAction<ConfigForm>
+    {
+        @Override
+        public void addNavTrail(NavTree root)
+        {
+            root.addChild("Query-based specimen import");
+        }
+
+        @Override
+        public void validateCommand(ConfigForm target, Errors errors)
+        {
+        }
+
+        @Override
+        public ModelAndView getView(ConfigForm form, boolean reshow, BindException errors)
+        {
+            return new JspView<>("/org/labkey/specimen/view/configureQueryImport.jsp", form, errors);
+        }
+
+        @Override
+        public boolean handlePost(ConfigForm form, BindException errors)
+        {
+            String QBSpecimenImportKey = QueryBasedSpecimenTransform.PROPERTY_MAP_KEY;
+            WritablePropertyMap props = PropertyManager.getWritableProperties(getContainer(), QBSpecimenImportKey, true);
+            form.setUserId(getUser().getUserId());
+            Map<String, String> valuesToPersist = form.getOptions();
+
+            if (!valuesToPersist.isEmpty())
+            {
+                props.putAll(valuesToPersist);
+                props.save();
+                return true;
+            }
+
+            return false;
+        }
+
+        @Override
+        public URLHelper getSuccessURL(ConfigForm configForm)
+        {
+            return PageFlowUtil.urlProvider(StudyUrls.class).getManageStudyURL(getContainer());
+        }
+    }
+
+    @RequiresPermission(AdminPermission.class)
+    public static class ReloadQueryBasedImportAction extends MutatingApiAction<ConfigForm>
+    {
+        @Override
+        public Object execute(ConfigForm form, BindException errors) throws Exception
+        {
+            ApiSimpleResponse response = new ApiSimpleResponse();
+
+            try
+            {
+                SpecimenTransform transform = SpecimenService.get().getSpecimenTransform(QueryBasedSpecimenTransform.NAME);
+                PipelineJob job = SpecimenService.get().createSpecimenReloadJob(getContainer(), getUser(), transform, getViewContext().getActionURL());
+
+                PipelineService.get().queueJob(job);
+                response.put("success", true);
+            }
+            catch (PipelineValidationException e)
+            {
+                throw new IOException(e);
+            }
+            return response;
+        }
+    }
 }

specimen/src/org/labkey/specimen/importer/AbstractSpecimenTask.java

@@ -132,22 +132,25 @@ public static void doImport(@Nullable FileLike inputFile, PipelineJob job, Simpl
             SpecimenImporter importer = new SpecimenImporter(ctx.getContainer(), ctx.getUser());
             importer.process(specimenDir, merge, ctx, job, syncParticipantVisit);
 
-            // perform any tasks after the transform and import has been completed
-            String activeImporter = SpecimenService.get().getActiveSpecimenImporter(ctx.getContainer());
-            if (null != activeImporter)
-            {
-                SpecimenTransform activeTransformer = SpecimenService.get().getSpecimenTransform(activeImporter);
-                if (activeTransformer != null && activeTransformer.getFileType().isType(inputFile))
-                    doPostTransform(activeTransformer, inputFile, job);
-            }
-            else
+            // if there's an inputFile, perform any tasks after the transform and import has been completed
+            if (inputFile != null)
             {
-                for (SpecimenTransform transformer : SpecimenService.get().getSpecimenTransforms(ctx.getContainer()))
+                String activeImporter = SpecimenService.get().getActiveSpecimenImporter(ctx.getContainer());
+                if (null != activeImporter)
                 {
-                    if (transformer.getFileType().isType(inputFile))
+                    SpecimenTransform activeTransformer = SpecimenService.get().getSpecimenTransform(activeImporter);
+                    if (activeTransformer != null && activeTransformer.getFileType().isType(inputFile))
+                        doPostTransform(activeTransformer, inputFile, job);
+                }
+                else
+                {
+                    for (SpecimenTransform transformer : SpecimenService.get().getSpecimenTransforms(ctx.getContainer()))
                     {
-                        doPostTransform(transformer, inputFile, job);
-                        break;
+                        if (transformer.getFileType().isType(inputFile))
+                        {
+                            doPostTransform(transformer, inputFile, job);
+                            break;
+                        }
                     }
                 }
             }

specimen/src/org/labkey/specimen/importer/QueryBasedExport.java

@@ -0,0 +1,97 @@
+package org.labkey.specimen.importer;
+
+import org.labkey.api.action.NullSafeBindException;
+import org.labkey.api.data.ColumnHeaderType;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.ShowRows;
+import org.labkey.api.data.TSVGridWriter;
+import org.labkey.api.data.TSVWriter;
+import org.labkey.api.pipeline.PipelineJob;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.query.QueryService;
+import org.labkey.api.query.QuerySettings;
+import org.labkey.api.query.QueryView;
+import org.labkey.api.query.UserSchema;
+import org.labkey.api.security.User;
+import org.labkey.api.view.ActionURL;
+import org.labkey.api.view.ViewContext;
+import org.labkey.specimen.actions.SpecimenController.ConfigureQueryImportAction;
+import org.labkey.vfs.FileLike;
+import org.springframework.validation.BindException;
+import org.springframework.validation.ObjectError;
+
+import java.io.IOException;
+import java.util.List;
+
+public class QueryBasedExport
+{
+    private final QueryBasedSpecimenReloadConfig _config;
+    private final PipelineJob _job;
+    private final FileLike _archive;
+
+    public QueryBasedExport (QueryBasedSpecimenReloadConfig config, PipelineJob job, FileLike archive)
+    {
+        _config = config;
+        _job = job;
+        _archive = archive;
+    }
+
+    public QueryBasedSpecimenReloadConfig getConfig()
+    {
+        return _config;
+    }
+
+    public void exportRepository() throws PipelineJobException
+    {
+        _job.info("Starting query-based specimen export");
+
+        User user = _job.getUser();
+        Container c = _job.getContainer();
+        String schemaName = _config.getSchemaName();
+        String queryName = _config.getQueryName();
+        String viewName = _config.getViewName();
+
+        // We fake up a ViewContext here as we're in a background job, and and no request-based ViewContext is available
+        try (ViewContext.StackResetter reset = ViewContext.pushMockViewContext(user, c, new ActionURL(ConfigureQueryImportAction.class, c)))
+        {
+            ViewContext viewContext = reset.getContext();
+
+            UserSchema userSchema = QueryService.get().getUserSchema(user, c, schemaName);
+            if (null == userSchema)
+                throw new PipelineJobException(String.format("Schema %s is either inaccessible or deleted.", schemaName));
+
+            QuerySettings querySettings = new QuerySettings(viewContext, "query", queryName);
+            querySettings.setSchemaName(schemaName);
+            querySettings.setViewName(viewName);
+            querySettings.setShowRows(ShowRows.ALL);
+
+            _job.info(String.format("Requesting data from query '%s' of schema '%s'", queryName, schemaName));
+            BindException errors = new NullSafeBindException(new Object(), "form");
+            QueryView queryView = userSchema.createView(viewContext, querySettings, errors);
+
+            if (errors.hasErrors())
+            {
+                List<ObjectError> allErrors = errors.getAllErrors();
+                for (ObjectError error : allErrors)
+                {
+                    _job.error(error.toString());
+                }
+                return;
+            }
+
+            _job.info("creating the exported data .csv file");
+            try (TSVGridWriter tsvWriter = queryView.getTsvWriter())
+            {
+                tsvWriter.setDelimiterCharacter(TSVWriter.DELIM.COMMA);
+                tsvWriter.setColumnHeaderType(ColumnHeaderType.DisplayFieldKey);
+                tsvWriter.write(_archive.openOutputStream());
+                _job.info("finished writing data file: " + _archive.getName());
+            }
+            catch (IOException e)
+            {
+                _job.error("Error writing TSV: " + e.getMessage());
+            }
+        }
+        _job.info("Finished writing data file: " + _archive.getName()); // use Format
+    }
+}