diff --git a/specimen/src/org/labkey/specimen/SpecimenModule.java b/specimen/src/org/labkey/specimen/SpecimenModule.java index f4baed4e00a..833af41d63d 100644 --- a/specimen/src/org/labkey/specimen/SpecimenModule.java +++ b/specimen/src/org/labkey/specimen/SpecimenModule.java @@ -57,6 +57,7 @@ import org.labkey.api.study.importer.SimpleStudyImporterRegistry; import org.labkey.api.study.writer.SimpleStudyWriterRegistry; import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.SystemMaintenance; import org.labkey.api.util.emailTemplate.EmailTemplateService; import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.ActionURL; @@ -66,6 +67,8 @@ import org.labkey.specimen.actions.SpecimenApiController; import org.labkey.specimen.actions.SpecimenController; import org.labkey.specimen.importer.AbstractSpecimenTask; +import org.labkey.specimen.importer.QueryBasedSpecimenImportUploadTask; +import org.labkey.specimen.importer.QueryBasedSpecimenTransform; import org.labkey.specimen.importer.RequestabilityManager; import org.labkey.specimen.importer.SpecimenImporter; import org.labkey.specimen.importer.SpecimenSchemaImporter; @@ -232,7 +235,9 @@ public void addSpecimenPivotTableNames(Set names) return null; } }); - } + + SystemMaintenance.addTask(new QueryBasedSpecimenImportUploadTask()); + } @Override protected void startupAfterSpringConfig(ModuleContext moduleContext) @@ -254,6 +259,10 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) new SpecimenSettingsImporter() )); + SpecimenService specimenService = SpecimenService.get(); + if (null != specimenService) + specimenService.registerSpecimenTransform(new QueryBasedSpecimenTransform()); + UsageMetricsService svc = UsageMetricsService.get(); if (null != svc) { diff --git a/specimen/src/org/labkey/specimen/actions/SpecimenController.java b/specimen/src/org/labkey/specimen/actions/SpecimenController.java index 2d05110bb03..e2fd64a0701 100644 --- a/specimen/src/org/labkey/specimen/actions/SpecimenController.java +++ b/specimen/src/org/labkey/specimen/actions/SpecimenController.java @@ -64,8 +64,10 @@ import org.labkey.api.module.FolderType; import org.labkey.api.module.Module; import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.PipelineStatusUrls; +import org.labkey.api.pipeline.PipelineValidationException; import org.labkey.api.pipeline.browse.PipelinePathForm; import org.labkey.api.query.AbstractQueryImportAction; import org.labkey.api.query.BatchValidationException; @@ -152,6 +154,7 @@ import org.labkey.specimen.SpecimenRequestException; import org.labkey.specimen.SpecimenRequestManager; import org.labkey.specimen.SpecimenRequestStatus; +import org.labkey.specimen.importer.QueryBasedSpecimenTransform; import org.labkey.specimen.importer.RequestabilityManager; import org.labkey.specimen.importer.SimpleSpecimenImporter; import org.labkey.specimen.model.ExtendedSpecimenRequestView; @@ -195,7 +198,6 @@ import java.io.BufferedOutputStream; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; @@ -5759,4 +5761,143 @@ public void addNavTrail(NavTree root) root.addChild("Insert " + _form.getQueryName()); } } + + public static class ConfigForm + { + private String _schemaName; + private String _queryName; + private String _viewName; + private String _enabled; + private int _userId; + + public int getUserId() + { + return _userId; + } + + public void setUserId(int userId) + { + _userId = userId; + } + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public String getQueryName() + { + return _queryName; + } + + public void setQueryName(String queryName) + { + _queryName = queryName; + } + + public String getViewName() + { + return _viewName; + } + + public void setViewName(String viewName) + { + _viewName = viewName; + } + + public String getEnabled() + { + return _enabled; + } + + public void setEnabled(String enabled) + { + _enabled = enabled; + } + + public Map getOptions() + { + Map valueMap = new HashMap<>(); + valueMap.put("schemaName", _schemaName); + valueMap.put("queryName", _queryName); + valueMap.put("viewName", _viewName); + valueMap.put("enabled", _enabled); + valueMap.put("userId", String.valueOf(_userId)); + return valueMap; + } + } + + @RequiresPermission(AdminPermission.class) + public static class ConfigureQueryImportAction extends FormViewAction + { + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Query-based specimen import"); + } + + @Override + public void validateCommand(ConfigForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(ConfigForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/specimen/view/configureQueryImport.jsp", form, errors); + } + + @Override + public boolean handlePost(ConfigForm form, BindException errors) + { + String QBSpecimenImportKey = QueryBasedSpecimenTransform.PROPERTY_MAP_KEY; + WritablePropertyMap props = PropertyManager.getWritableProperties(getContainer(), QBSpecimenImportKey, true); + form.setUserId(getUser().getUserId()); + Map valuesToPersist = form.getOptions(); + + if (!valuesToPersist.isEmpty()) + { + props.putAll(valuesToPersist); + props.save(); + return true; + } + + return false; + } + + @Override + public URLHelper getSuccessURL(ConfigForm configForm) + { + return PageFlowUtil.urlProvider(StudyUrls.class).getManageStudyURL(getContainer()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ReloadQueryBasedImportAction extends MutatingApiAction + { + @Override + public Object execute(ConfigForm form, BindException errors) throws Exception + { + ApiSimpleResponse response = new ApiSimpleResponse(); + + try + { + SpecimenTransform transform = SpecimenService.get().getSpecimenTransform(QueryBasedSpecimenTransform.NAME); + PipelineJob job = SpecimenService.get().createSpecimenReloadJob(getContainer(), getUser(), transform, getViewContext().getActionURL()); + + PipelineService.get().queueJob(job); + response.put("success", true); + } + catch (PipelineValidationException e) + { + throw new IOException(e); + } + return response; + } + } } diff --git a/specimen/src/org/labkey/specimen/importer/AbstractSpecimenTask.java b/specimen/src/org/labkey/specimen/importer/AbstractSpecimenTask.java index 3acf4e95dda..951b261b880 100644 --- a/specimen/src/org/labkey/specimen/importer/AbstractSpecimenTask.java +++ b/specimen/src/org/labkey/specimen/importer/AbstractSpecimenTask.java @@ -132,22 +132,25 @@ public static void doImport(@Nullable FileLike inputFile, PipelineJob job, Simpl SpecimenImporter importer = new SpecimenImporter(ctx.getContainer(), ctx.getUser()); importer.process(specimenDir, merge, ctx, job, syncParticipantVisit); - // perform any tasks after the transform and import has been completed - String activeImporter = SpecimenService.get().getActiveSpecimenImporter(ctx.getContainer()); - if (null != activeImporter) - { - SpecimenTransform activeTransformer = SpecimenService.get().getSpecimenTransform(activeImporter); - if (activeTransformer != null && activeTransformer.getFileType().isType(inputFile)) - doPostTransform(activeTransformer, inputFile, job); - } - else + // if there's an inputFile, perform any tasks after the transform and import has been completed + if (inputFile != null) { - for (SpecimenTransform transformer : SpecimenService.get().getSpecimenTransforms(ctx.getContainer())) + String activeImporter = SpecimenService.get().getActiveSpecimenImporter(ctx.getContainer()); + if (null != activeImporter) { - if (transformer.getFileType().isType(inputFile)) + SpecimenTransform activeTransformer = SpecimenService.get().getSpecimenTransform(activeImporter); + if (activeTransformer != null && activeTransformer.getFileType().isType(inputFile)) + doPostTransform(activeTransformer, inputFile, job); + } + else + { + for (SpecimenTransform transformer : SpecimenService.get().getSpecimenTransforms(ctx.getContainer())) { - doPostTransform(transformer, inputFile, job); - break; + if (transformer.getFileType().isType(inputFile)) + { + doPostTransform(transformer, inputFile, job); + break; + } } } } diff --git a/specimen/src/org/labkey/specimen/importer/QueryBasedExport.java b/specimen/src/org/labkey/specimen/importer/QueryBasedExport.java new file mode 100644 index 00000000000..7dde054b175 --- /dev/null +++ b/specimen/src/org/labkey/specimen/importer/QueryBasedExport.java @@ -0,0 +1,97 @@ +package org.labkey.specimen.importer; + +import org.labkey.api.action.NullSafeBindException; +import org.labkey.api.data.ColumnHeaderType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ShowRows; +import org.labkey.api.data.TSVGridWriter; +import org.labkey.api.data.TSVWriter; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.ViewContext; +import org.labkey.specimen.actions.SpecimenController.ConfigureQueryImportAction; +import org.labkey.vfs.FileLike; +import org.springframework.validation.BindException; +import org.springframework.validation.ObjectError; + +import java.io.IOException; +import java.util.List; + +public class QueryBasedExport +{ + private final QueryBasedSpecimenReloadConfig _config; + private final PipelineJob _job; + private final FileLike _archive; + + public QueryBasedExport (QueryBasedSpecimenReloadConfig config, PipelineJob job, FileLike archive) + { + _config = config; + _job = job; + _archive = archive; + } + + public QueryBasedSpecimenReloadConfig getConfig() + { + return _config; + } + + public void exportRepository() throws PipelineJobException + { + _job.info("Starting query-based specimen export"); + + User user = _job.getUser(); + Container c = _job.getContainer(); + String schemaName = _config.getSchemaName(); + String queryName = _config.getQueryName(); + String viewName = _config.getViewName(); + + // We fake up a ViewContext here as we're in a background job, and and no request-based ViewContext is available + try (ViewContext.StackResetter reset = ViewContext.pushMockViewContext(user, c, new ActionURL(ConfigureQueryImportAction.class, c))) + { + ViewContext viewContext = reset.getContext(); + + UserSchema userSchema = QueryService.get().getUserSchema(user, c, schemaName); + if (null == userSchema) + throw new PipelineJobException(String.format("Schema %s is either inaccessible or deleted.", schemaName)); + + QuerySettings querySettings = new QuerySettings(viewContext, "query", queryName); + querySettings.setSchemaName(schemaName); + querySettings.setViewName(viewName); + querySettings.setShowRows(ShowRows.ALL); + + _job.info(String.format("Requesting data from query '%s' of schema '%s'", queryName, schemaName)); + BindException errors = new NullSafeBindException(new Object(), "form"); + QueryView queryView = userSchema.createView(viewContext, querySettings, errors); + + if (errors.hasErrors()) + { + List allErrors = errors.getAllErrors(); + for (ObjectError error : allErrors) + { + _job.error(error.toString()); + } + return; + } + + _job.info("creating the exported data .csv file"); + try (TSVGridWriter tsvWriter = queryView.getTsvWriter()) + { + tsvWriter.setDelimiterCharacter(TSVWriter.DELIM.COMMA); + tsvWriter.setColumnHeaderType(ColumnHeaderType.DisplayFieldKey); + tsvWriter.write(_archive.openOutputStream()); + _job.info("finished writing data file: " + _archive.getName()); + } + catch (IOException e) + { + _job.error("Error writing TSV: " + e.getMessage()); + } + } + _job.info("Finished writing data file: " + _archive.getName()); // use Format + } +} diff --git a/specimen/src/org/labkey/specimen/importer/QueryBasedSpecimenImportUploadTask.java b/specimen/src/org/labkey/specimen/importer/QueryBasedSpecimenImportUploadTask.java new file mode 100644 index 00000000000..af9b7f64b7f --- /dev/null +++ b/specimen/src/org/labkey/specimen/importer/QueryBasedSpecimenImportUploadTask.java @@ -0,0 +1,81 @@ +package org.labkey.specimen.importer; + +import org.apache.logging.log4j.Logger; +import org.labkey.api.data.Container; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.study.SpecimenService; +import org.labkey.api.study.SpecimenTransform; +import org.labkey.api.study.StudyService; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.specimen.SpecimenModule; + +import java.util.stream.Stream; + +public class QueryBasedSpecimenImportUploadTask implements SystemMaintenance.MaintenanceTask +{ + @Override + public String getDescription() + { + return "Query-Based Specimen Import Upload Task"; + } + + @Override + public String getName() + { + return "QueryBasedSpecimenUpload"; + } + + @Override + public void run(Logger log) + { + String QBSpecimenImportKey = QueryBasedSpecimenTransform.PROPERTY_MAP_KEY; + try (Stream stream = PropertyManager.getNormalStore().streamMatchingContainers(PropertyManager.SHARED_USER, QBSpecimenImportKey)) + { + stream.forEach(c -> { + try + { + // Study must have been deleted + if (null == StudyService.get().getStudy(c)) + { + log.error("Query-based specimen import failed: Study does not exist in folder {}", c.getPath()); + return; + } + + if (!c.getActiveModules().contains(ModuleLoader.getInstance().getModule(SpecimenModule.class))) + return; + + SpecimenTransform transform = SpecimenService.get().getSpecimenTransform(QueryBasedSpecimenTransform.NAME); + PropertyManager.PropertyMap props = PropertyManager.getProperties(c, QBSpecimenImportKey); + boolean enabled = ("on").equals(props.get("enabled")); + + if (!enabled) + { + log.info("Prohibiting queuing specimen import for {}. Query-based specimen import is not enabled.", c.getName()); + return; + } + if (!transform.isActive(c)) + { + log.info("Prohibiting queuing specimen import for {}. Query-based specimen import is not the active import mechanism.", c.getName()); + return; + } + log.info("Queuing specimen import for {}", c.getName()); + + int userId = Integer.parseInt(props.get("userId")); + User reloadUser = UserManager.getUser(userId); + PipelineJob job = SpecimenService.get().createSpecimenReloadJob(c, reloadUser, transform, null); + + PipelineService.get().queueJob(job); + } + catch (Exception e) + { + log.error("Query-based specimen import failed", e); + } + }); + } + } +} diff --git a/specimen/src/org/labkey/specimen/importer/QueryBasedSpecimenReloadConfig.java b/specimen/src/org/labkey/specimen/importer/QueryBasedSpecimenReloadConfig.java new file mode 100644 index 00000000000..396d63c6859 --- /dev/null +++ b/specimen/src/org/labkey/specimen/importer/QueryBasedSpecimenReloadConfig.java @@ -0,0 +1,58 @@ +package org.labkey.specimen.importer; + +import org.labkey.api.study.SpecimenTransform; + +public class QueryBasedSpecimenReloadConfig implements SpecimenTransform.ExternalImportConfig +{ + private String schemaName; + private String queryName; + private String viewName; + + public String getSchemaName() + { + return schemaName; + } + + public void setSchemaName(String schemaName) + { + this.schemaName = schemaName; + } + + public String getQueryName() + { + return queryName; + } + + public void setQueryName(String queryName) + { + this.queryName = queryName; + } + + public String getViewName() + { + return viewName; + } + + public void setViewName(String viewName) + { + this.viewName = viewName; + } + + @Override + public String getBaseServerUrl() + { + return null; + } + + @Override + public String getUsername() + { + return null; + } + + @Override + public String getPassword() + { + return null; + } +} diff --git a/specimen/src/org/labkey/specimen/importer/QueryBasedSpecimenTransform.java b/specimen/src/org/labkey/specimen/importer/QueryBasedSpecimenTransform.java new file mode 100644 index 00000000000..1e92c7b54bf --- /dev/null +++ b/specimen/src/org/labkey/specimen/importer/QueryBasedSpecimenTransform.java @@ -0,0 +1,128 @@ +package org.labkey.specimen.importer; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.TSVMapWriter; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.security.User; +import org.labkey.api.study.SpecimenService; +import org.labkey.api.study.SpecimenTransform; +import org.labkey.api.util.FileType; +import org.labkey.api.view.ActionURL; +import org.labkey.specimen.SpecimenModule; +import org.labkey.specimen.actions.SpecimenController.ConfigureQueryImportAction; +import org.labkey.vfs.FileLike; + +import java.util.ArrayList; +import java.util.Map; + +public class QueryBasedSpecimenTransform implements SpecimenTransform +{ + public static final String NAME = "QueryBased"; + public static final String PROPERTY_MAP_KEY = "queryBasedSpecimenLoader"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public boolean isValid(Container container) + { + return container.getActiveModules().contains(ModuleLoader.getInstance().getModule(SpecimenModule.class)); + } + + @Override + public boolean isActive(Container container) + { + PropertyManager.PropertyMap props = PropertyManager.getProperties(container, PROPERTY_MAP_KEY); + boolean enabled = ("on").equals(props.get("enabled")); + // If selected is null, the container has only one transform + String selected = SpecimenService.get().getActiveSpecimenImporter(container); + + return enabled && (selected == null || NAME.equals(selected)); + } + + @Override + public FileType getFileType() + { + return new FileType(".qbst.csv"); + } + + protected static class QueryBasedTSVWriter extends TSVMapWriter + { + private int _rowCount; + public QueryBasedTSVWriter(Iterable> rows) + { + super(new ArrayList<>(), rows); // start with empty column list + } + + public int getRowCount() + { + return _rowCount; + } + + @Override + public void writeRow(Map row) + { + if (_rowCount == 0) + { + writeFileHeader(); + setColumns(row.keySet()); + writeColumnHeaders(); + } + super.writeRow(row); + _rowCount++; + } + } + + @Override + public void transform(@Nullable PipelineJob job, FileLike input, FileLike output) throws PipelineJobException + { + QueryBasedTransformTask task = new QueryBasedTransformTask(job); + task.transform(input, output); + } + + @Override + public void postTransform(@Nullable PipelineJob job, FileLike input, FileLike outputArchive) + { + // noop + } + + @Override + public @Nullable ActionURL getManageAction(Container c, User user) + { + return new ActionURL(ConfigureQueryImportAction.class, c); + } + + @Override + public ExternalImportConfig getExternalImportConfig(Container c, User user) + { + Map props = PropertyManager.getProperties(c, PROPERTY_MAP_KEY); + QueryBasedSpecimenReloadConfig config = new QueryBasedSpecimenReloadConfig(); + + String schemaName = props.get("schemaName"); + String queryName = props.get("queryName"); + String viewName = props.get("viewName"); + + config.setSchemaName(schemaName); + config.setQueryName(queryName); + config.setViewName(viewName); + + return config; + } + + @Override + public void importFromExternalSource(@Nullable PipelineJob job, ExternalImportConfig importConfig, FileLike inputArchive) throws PipelineJobException + { + if (importConfig instanceof QueryBasedSpecimenReloadConfig queryConfig) + { + QueryBasedExport export = new QueryBasedExport(queryConfig, job, inputArchive); + export.exportRepository(); + } + } +} diff --git a/specimen/src/org/labkey/specimen/importer/QueryBasedTransformTask.java b/specimen/src/org/labkey/specimen/importer/QueryBasedTransformTask.java new file mode 100644 index 00000000000..90a61d1c2b5 --- /dev/null +++ b/specimen/src/org/labkey/specimen/importer/QueryBasedTransformTask.java @@ -0,0 +1,160 @@ +package org.labkey.specimen.importer; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.pipeline.AbstractSpecimenTransformTask; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.reader.DataLoader; +import org.labkey.api.reader.DataLoaderFactory; +import org.labkey.api.writer.PrintWriters; +import org.labkey.vfs.FileLike; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class QueryBasedTransformTask extends AbstractSpecimenTransformTask +{ + public QueryBasedTransformTask(@Nullable PipelineJob job) + { + super(job); + } + + private final Map _primaryIds = new LinkedHashMap<>(); + private final Map _derivativeIds = new LinkedHashMap<>(); + private final Map _additiveIds = new LinkedHashMap<>(); + + @Nullable + private String getPrimaryType(Map row) + { + String colNameLK = getNonNullValue(row, "primary_type"); + return ("".equals(colNameLK)) ? getNonNullValue(row, "PrimaryType") : colNameLK; + } + + @Nullable + private String getDerivative(Map row) + { + String colNameLK = getNonNullValue(row, "derivative_type"); + return ("".equals(colNameLK)) ? getNonNullValue(row, "DerivativeType") : colNameLK; + } + + @Nullable + private String getAdditive(Map row) + { + String colNameLK = getNonNullValue(row, "additive_type"); + return ("".equals(colNameLK)) ? getNonNullValue(row, "AdditiveType") : colNameLK; + } + + private Integer getType(String type, Map typeMap) + { + Integer id = typeMap.get(type); + if (id == null && !StringUtils.isEmpty(type)) + { + id = typeMap.size() + 1; + typeMap.put(type, id); + } + return id; + } + + public void transform(FileLike input, FileLike output) throws PipelineJobException + { + info("Starting to transform input file " + input + " to output file " + output); + + try + { + DataLoaderFactory df = DataLoader.get().findFactory(input, null); + if (null == df) + throw new PipelineJobException("Unable to create a data loader factory for the file: " + input.getName()); + DataLoader loader = df.createLoader(input.openInputStream(), true, _job.getContainer()); + loader.setInferTypes(false); + + try (ZipOutputStream zOut = new ZipOutputStream(output.openOutputStream())) + { + zOut.putNextEntry(new ZipEntry("specimens.tsv")); + PrintWriter writer = PrintWriters.getPrintWriter(zOut); + final QueryBasedSpecimenTransform.QueryBasedTSVWriter tsvWriter = new QueryBasedSpecimenTransform.QueryBasedTSVWriter(Collections.emptyList()); + + tsvWriter.setFileHeader(Collections.singletonList("# " + "specimens")); + tsvWriter.setPrintWriter(writer); + try + { + loader.forEach(row -> { + if (MapFilter.getMapFilter(this).test(row)) + { + if (!row.containsKey("records_id")) + row.put("record_id", tsvWriter.getRowCount()); + + row.put("primary_specimen_type_id", getType(getPrimaryType(row), _primaryIds)); + row.put("derivative_type_id", getType(getDerivative(row), _derivativeIds)); + row.put("additive_type_id", getType(getAdditive(row), _additiveIds)); + + tsvWriter.writeRow(row); + } + }); + } + finally + { + writer.flush(); + zOut.closeEntry(); + } + + if (tsvWriter.getRowCount() > 0) + info("After removing duplicates, there are " + tsvWriter.getRowCount() + " rows of data"); + else + throw new PipelineJobException("There are no rows of data"); + + writePrimaries(getPrimaryIds(), zOut); + writeDerivatives(getDerivativeIds(), zOut); + writeAdditives(getAdditiveIds(), zOut); + } + } + + catch (IOException e) + { + throw new PipelineJobException(e); + } + } + + @Override + protected Map transformRow(Map inputRow, int rowIndex, Map labIds, Map primaryIds, Map derivativeIds) + { + return null; + } + + @Override + protected Set getIgnoredHashColumns() + { + return new CaseInsensitiveHashSet(); + } + + @Override + protected Map getLabIds() + { + return new LinkedHashMap<>(); + } + + @Override + protected Map getPrimaryIds() + { + return _primaryIds; + } + + @Override + protected Map getDerivativeIds() + { + return _derivativeIds; + } + + @Override + protected Map getAdditiveIds() + { + return _additiveIds; + } +} diff --git a/specimen/src/org/labkey/specimen/view/chooseImporter.jsp b/specimen/src/org/labkey/specimen/view/chooseImporter.jsp index 6cc910506b2..0fc794dd987 100644 --- a/specimen/src/org/labkey/specimen/view/chooseImporter.jsp +++ b/specimen/src/org/labkey/specimen/view/chooseImporter.jsp @@ -5,9 +5,7 @@ */ %> <%@ page import="com.google.common.collect.Iterables" %> -<%@ page import="org.labkey.api.admin.AdminUrls" %> <%@ page import="org.labkey.api.data.Container" %> -<%@ page import="org.labkey.api.module.ModuleLoader" %> <%@ page import="org.labkey.api.security.User" %> <%@ page import="org.labkey.api.study.SpecimenService" %> <%@ page import="org.labkey.api.study.SpecimenTransform" %> @@ -18,7 +16,6 @@ <%@ page import="org.labkey.api.view.ActionURL" %> <%@ page import="org.labkey.specimen.actions.ShowUploadSpecimensAction" %> <%@ page import="java.util.Collection" %> -<%@ page import="static org.labkey.api.util.HtmlString.NBSP" %> <%@ page extends="org.labkey.api.jsp.JspBase" %> <%@ taglib prefix="labkey" uri="http://www.labkey.org/taglib" %> <% @@ -34,9 +31,6 @@ int rowNumber = 0; String selected = SpecimenService.get().getActiveSpecimenImporter(c); - HtmlString manageFoldersLink = h(urlProvider(AdminUrls.class).getFolderTypeURL(c)); - HtmlString labkeyEditionsLink = h("https://www.labkey.com/products-services/labkey-server/labkey-server-editions-feature-comparison/"); - HtmlString contactUsLink = h("https://www.labkey.com/about/contact/"); HtmlString manuallyImportSpecimensLink = h(urlFor(ShowUploadSpecimensAction.class)); %> @@ -49,6 +43,12 @@ +<% + // At the moment, there's exactly one SpecimenTransform, QueryBasedSpecimenTransform, which is provided by the + // Specimen module. As a result, this "choose importer" page is never linked (if a single transform exists, the + // "Configure Specimen Import" link navigates straight its configuration page. This probably won't ever change, + // but we'll leave this page in place just in case. +%>
<% if (numberOfTransforms > 1) { %>

@@ -114,8 +114,7 @@ %> - <% } else if (numberOfTransforms == 1) { %> - <% + <% } else if (numberOfTransforms == 1) { SpecimenTransform transform = Iterables.get(specimenTransforms, 0); ActionURL manageAction = transform.getManageAction(c, user); %> @@ -138,21 +137,6 @@ <% } else { %> - <% if (ModuleLoader.getInstance().hasModule("professional")) { %> -

- External Specimen Import is not currently available for this folder. - To use External import, > enable the Professional Module for this folder. -
- <% } else { %> -
-

<%=NBSP%>

- External Specimen Import is a Premium LabKey feature. > Learn more or - > contact LabKey . -
- <% - } - %> - > Import specimens manually <% } diff --git a/specimen/src/org/labkey/specimen/view/configureQueryImport.jsp b/specimen/src/org/labkey/specimen/view/configureQueryImport.jsp new file mode 100644 index 00000000000..f62ceaeaa04 --- /dev/null +++ b/specimen/src/org/labkey/specimen/view/configureQueryImport.jsp @@ -0,0 +1,132 @@ +<% +/* + * Copyright (c) 2013-2016 LabKey Corporation. All rights reserved. No portion of this work may be reproduced in + * any form or by any electronic or mechanical means without written permission from LabKey Corporation. + */ +%> +<%@ page import="org.labkey.api.data.Container" %> +<%@ page import="org.labkey.api.data.PropertyManager" %> +<%@ page import="org.labkey.api.pipeline.PipelineStatusUrls" %> +<%@ page import="org.labkey.api.study.SpecimenService" %> +<%@ page import="org.labkey.api.study.StudyUrls" %> +<%@ page import="org.labkey.api.util.ButtonBuilder" %> +<%@ page import="org.labkey.api.util.InputBuilder" %> +<%@ page import="org.labkey.api.util.InputBuilder.Input" %> +<%@ page import="org.labkey.api.util.PageFlowUtil" %> +<%@ page import="org.labkey.api.util.SelectBuilder" %> +<%@ page import="org.labkey.specimen.actions.SpecimenController.ReloadQueryBasedImportAction" %> +<%@ page import="org.labkey.specimen.importer.QueryBasedSpecimenTransform" %> +<%@ page extends="org.labkey.api.jsp.JspBase" %> +<%@ taglib prefix="labkey" uri="http://www.labkey.org/taglib" %> + +<% + Container c = getContainer(); + String selected = SpecimenService.get().getActiveSpecimenImporter(c); + boolean active = selected == null || selected.equals(QueryBasedSpecimenTransform.NAME); + String QBSpecimenImportKey = QueryBasedSpecimenTransform.PROPERTY_MAP_KEY; + + PropertyManager.PropertyMap props = PropertyManager.getProperties(c, QBSpecimenImportKey); + + String schemaName = props.get("schemaName"); + String queryName = props.get("queryName"); + String viewName = props.get("viewName"); + boolean enabled = ("on").equals(props.get("enabled")); +%> + + + + + +
+

+ Specimen data can be automatically loaded from an existing query using the specimen import configuration + defined on this page. The schema you specify must already be defined as an external schema, and the query + and view selected must map and filter rows as expected by the LabKey specimen repository. +

+ +

+ Learn more about configuring <%=helpLink("querySpecimenImport", "Query-based specimen reload")%>. +

+ +
+ + +

Configure connection

+ +
+ <%=InputBuilder.checkbox() + .name("enabled") + .label("Enable reload") + .layout(Input.Layout.HORIZONTAL) + .checked(enabled) + .formGroup(true) + %> + + <%= new SelectBuilder() + .name("schemaName") + .id("schemaName") + .label("Schema") + .layout(Input.Layout.HORIZONTAL) + .formGroup(true) + %> + + <%= new SelectBuilder() + .name("queryName") + .id("queryName") + .label("Query") + .layout(Input.Layout.HORIZONTAL) + .formGroup(true) + %> + + <%= new SelectBuilder() + .name("viewName") + .id("viewName") + .label("View") + .layout(Input.Layout.HORIZONTAL) + .formGroup(true) + %> +
+ + + <%= new ButtonBuilder("Cancel") + .href(PageFlowUtil.urlProvider(StudyUrls.class).getManageStudyURL(getContainer())) + .build() + %> + + <%= new ButtonBuilder("Save") + .submit(true) + .build() + %> + + <%= new ButtonBuilder("Reload Now") + .id("reloadNow") + .enabled(enabled && active) + .onClick("reloadNow();") + .build() + %> + + + +
diff --git a/specimen/test/sampledata/import/100_specimens.xlsx b/specimen/test/sampledata/import/100_specimens.xlsx new file mode 100644 index 00000000000..75b54324a2e Binary files /dev/null and b/specimen/test/sampledata/import/100_specimens.xlsx differ diff --git a/specimen/test/src/org/labkey/test/pages/specimen/ConfigureSpecimenImportPage.java b/specimen/test/src/org/labkey/test/pages/specimen/ConfigureSpecimenImportPage.java new file mode 100644 index 00000000000..7114172d0f5 --- /dev/null +++ b/specimen/test/src/org/labkey/test/pages/specimen/ConfigureSpecimenImportPage.java @@ -0,0 +1,96 @@ +package org.labkey.test.pages.specimen; + +import org.labkey.test.Locator; +import org.labkey.test.WebDriverWrapper; +import org.labkey.test.WebTestHelper; +import org.labkey.test.pages.LabKeyPage; +import org.labkey.test.pages.study.ManageStudyPage; +import org.labkey.test.util.DataRegionTable; +import org.labkey.test.util.PipelineStatusTable; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; + +import java.util.Optional; + +public class ConfigureSpecimenImportPage extends LabKeyPage +{ + public ConfigureSpecimenImportPage(WebDriver driver) + { + super(driver); + } + + public static ConfigureSpecimenImportPage beginAt(WebDriverWrapper webDriverWrapper) + { + return beginAt(webDriverWrapper, webDriverWrapper.getCurrentContainerPath()); + } + + public static ConfigureSpecimenImportPage beginAt(WebDriverWrapper webDriverWrapper, String containerPath) + { + webDriverWrapper.beginAt(WebTestHelper.buildURL("specimen", containerPath, "configureQueryImport")); + return new ConfigureSpecimenImportPage(webDriverWrapper.getDriver()); + } + + @Override + protected void waitForPage() + { + WebDriverWrapper.waitFor(()-> elementCache().schemaSelect.isEnabled(), + "The page did not become ready in time", WAIT_FOR_JAVASCRIPT); + } + + public ConfigureSpecimenImportPage configureQueryBasedImport(boolean enableReload, String schema, String query, String view) + { + setCheckbox(elementCache().reloadBoxElement, enableReload); + setFormElement(elementCache().schemaSelect, schema); + setFormElement(elementCache().querySelect, query); + setFormElement(elementCache().viewSelect, view); + return this; + } + + public ManageStudyPage clickSave() + { + clickAndWait(elementCache().saveButton); + return new ManageStudyPage(getDriver()); + } + + public ManageStudyPage clickCancel() + { + clickAndWait(elementCache().cancelButton); + return new ManageStudyPage(getDriver()); + } + + public boolean isReloadNowButtonPresent() + { + return elementCache().reloadNowButton().isPresent(); + } + + public DataRegionTable clickReloadNow() + { + doAndWaitForPageToLoad(()-> elementCache().reloadNowButton().get().click()); + return PipelineStatusTable.finder(getDriver()).waitFor(); + } + + @Override + protected ElementCache newElementCache() + { + return new ElementCache(); + } + + protected class ElementCache extends LabKeyPage.ElementCache + { + WebElement cancelButton = Locator.linkWithSpan("Cancel").findWhenNeeded(getDriver()); + WebElement saveButton = Locator.linkWithSpan("Save").findWhenNeeded(getDriver()); + Optional reloadNowButton() + { + return Locator.linkWithSpan("Reload Now").findOptionalElement(getDriver()); + } + + WebElement formContainer = Locator.tagWithClass("div", "QBSpecimenImportFormFields") + .findWhenNeeded(getDriver()).withTimeout(WAIT_FOR_JAVASCRIPT); + + WebElement reloadBoxElement = Locator.checkbox().withAttribute("name", "enabled") + .findWhenNeeded(formContainer); + WebElement schemaSelect = Locator.id("schemaName").findWhenNeeded(formContainer); + WebElement querySelect = Locator.id("queryName").findWhenNeeded(formContainer); + WebElement viewSelect = Locator.id("viewName").findWhenNeeded(formContainer); + } +} diff --git a/specimen/test/src/org/labkey/test/tests/specimen/QueryBasedSpecimenImportTest.java b/specimen/test/src/org/labkey/test/tests/specimen/QueryBasedSpecimenImportTest.java new file mode 100644 index 00000000000..8e6df16477a --- /dev/null +++ b/specimen/test/src/org/labkey/test/tests/specimen/QueryBasedSpecimenImportTest.java @@ -0,0 +1,114 @@ +package org.labkey.test.tests.specimen; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.TestFileUtils; +import org.labkey.test.categories.Git; +import org.labkey.test.categories.Specimen; +import org.labkey.test.pages.specimen.ConfigureSpecimenImportPage; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.hasItems; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +@Category({Git.class, Specimen.class}) +public class QueryBasedSpecimenImportTest extends BaseWebDriverTest +{ + String TEST_SCHEMA = "lists"; + String TEST_QUERY = "test_specimen_data"; + String TEST_VIEW = "Default"; + File SPECIMENS_FILE = TestFileUtils.getSampleData("import/100_specimens.xlsx"); + + @BeforeClass + public static void setupProject() + { + QueryBasedSpecimenImportTest init = getCurrentTest(); + init.doSetup(); + } + + private void doSetup() + { + _containerHelper.createProject(getProjectName(), "Study"); + _studyHelper.startCreateStudy() + .createStudy(); + goToFolderManagement() + .goToFolderTypeTab() + .enableModule("Specimen") + .save(); + + // create a list of specimens to import from + _listHelper.createListFromFile(getProjectName(), TEST_QUERY, SPECIMENS_FILE); + } + + @Before + public void preTest() + { + goToProjectHome(); + } + + @Test + public void testQueryBasedSpecimenImport() + { + // the test configures one query-based importer + goToManageStudy() + .clickConfigureSpecimenImport(); + new ConfigureSpecimenImportPage(getDriver()) + .configureQueryBasedImport(true, TEST_SCHEMA, TEST_QUERY, TEST_VIEW) + .clickSave(); + + // navigate directly to configure query-based import, user 'reload now' button to load the specimens from the query + ConfigureSpecimenImportPage.beginAt(this, getProjectName()) + .clickReloadNow(); + waitForPipelineJobsToComplete(1, false); + + // navigate to Specimens tab, verify + clickTab("Specimen Data", true); + + // iterate over the list, ensure that for each there is a matching specimen in specimenDetails + List> sourceSpecimenData =executeSelectRowCommand(TEST_SCHEMA, TEST_QUERY).getRows(); + List> specimenDetailsList = executeSelectRowCommand("study", "SpecimenDetail").getRows(); + List> specimenPrimaryTypes = executeSelectRowCommand("study", "SpecimenPrimaryType").getRows(); + + List primaryTypes = specimenPrimaryTypes.stream().map(a-> a.get("PrimaryType").toString()).collect(Collectors.toList()); + assertThat("expect 4 kinds of primary vial types", + primaryTypes, hasItems("PBMC-Na Hep", "Serum-Clot", "urine super", "whole blood")); + + for (Map specimenRow : sourceSpecimenData) + { + String globalUniqueSpecimenId = specimenRow.get("global_unique_specimen_id").toString(); + + Optional> matchingSpecimenDetail = specimenDetailsList.stream() + .filter(a-> a.get("GlobalUniqueId").equals(globalUniqueSpecimenId)).findFirst(); + assertTrue("Expect to find matching SpecimenDetail for [" +globalUniqueSpecimenId+ "]", + matchingSpecimenDetail.isPresent()); + } + } + + @Override + protected BrowserType bestBrowser() + { + return BrowserType.CHROME; + } + + @Override + protected String getProjectName() + { + return "SpecimenImportTest Project"; + } + + @Override + public List getAssociatedModules() + { + return Arrays.asList("Specimen"); + } +}