diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 961f9495a36..8a67c0fb3b0 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -10,6 +10,6 @@ \ No newline at end of file +--> diff --git a/announcements/src/org/labkey/announcements/announcementThread.jsp b/announcements/src/org/labkey/announcements/announcementThread.jsp index b636af6df12..1b8711bc415 100644 --- a/announcements/src/org/labkey/announcements/announcementThread.jsp +++ b/announcements/src/org/labkey/announcements/announcementThread.jsp @@ -127,7 +127,7 @@ if (!announcementModel.getAttachments().isEmpty()) { ActionURL downloadURL = AnnouncementsController.getDownloadURL(announcementModel, d.getName()); %> -  <%=h(d.getName())%> <% + <%=d.renderDownloadLink(downloadURL)%> <% } %> <% @@ -210,7 +210,7 @@ if (!announcementModel.getResponses().isEmpty()) { ActionURL downloadURL = AnnouncementsController.getDownloadURL(r, rd.getName()); %> -  <%=h(rd.getName())%> <% + <%=rd.renderDownloadLink(downloadURL)%> <% } %> diff --git a/announcements/src/org/labkey/announcements/announcementWebPartSimple.jsp b/announcements/src/org/labkey/announcements/announcementWebPartSimple.jsp index 713869f8ee8..dac75ebc99c 100644 --- a/announcements/src/org/labkey/announcements/announcementWebPartSimple.jsp +++ b/announcements/src/org/labkey/announcements/announcementWebPartSimple.jsp @@ -195,7 +195,7 @@ for (AnnouncementModel a : bean.announcementModels) for (Attachment d : a.getAttachments()) { ActionURL downloadURL = AnnouncementsController.getDownloadURL(a, d.getName()); - %> <%=h(d.getName())%> <% + %><%=d.renderDownloadLink(downloadURL)%> <% } %><% } diff --git a/announcements/src/org/labkey/announcements/announcementWebPartWithExpandos.jsp b/announcements/src/org/labkey/announcements/announcementWebPartWithExpandos.jsp index 4322dc4b7ed..8f061e75af7 100644 --- a/announcements/src/org/labkey/announcements/announcementWebPartWithExpandos.jsp +++ b/announcements/src/org/labkey/announcements/announcementWebPartWithExpandos.jsp @@ -217,7 +217,7 @@ for (AnnouncementModel a : bean.announcementModels) for (Attachment d : a.getAttachments()) { ActionURL downloadURL = AnnouncementsController.getDownloadURL(a, d.getName()); - %> <%=h(d.getName())%> <% + %><%=d.renderDownloadLink(downloadURL)%> <% } %><% } diff --git a/announcements/src/org/labkey/announcements/update.jsp b/announcements/src/org/labkey/announcements/update.jsp index 1797588ed54..c5e54ef947b 100644 --- a/announcements/src/org/labkey/announcements/update.jsp +++ b/announcements/src/org/labkey/announcements/update.jsp @@ -140,7 +140,6 @@ if (settings.hasExpires()) <% int x = -1; - String id; for (Attachment att : ann.getAttachments()) { x++; diff --git a/api/src/org/labkey/api/assay/AbstractAssayProvider.java b/api/src/org/labkey/api/assay/AbstractAssayProvider.java index 0f4b1988530..c620c02341b 100644 --- a/api/src/org/labkey/api/assay/AbstractAssayProvider.java +++ b/api/src/org/labkey/api/assay/AbstractAssayProvider.java @@ -124,6 +124,7 @@ import javax.script.ScriptEngine; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.URL; import java.sql.ResultSet; @@ -1274,9 +1275,9 @@ public Pair> setValidationAndAnalysisS if (!(engine instanceof ExternalScriptEngine && ((ExternalScriptEngine) engine).isBinary(scriptFile))) { String scriptText; - try + try (InputStream is = scriptFile.openInputStream()) { - scriptText = IOUtils.toString(scriptFile.openInputStream(), StringUtilsLabKey.DEFAULT_CHARSET); + scriptText = IOUtils.toString(is, StringUtilsLabKey.DEFAULT_CHARSET); } catch (IOException e) { diff --git a/api/src/org/labkey/api/attachments/Attachment.java b/api/src/org/labkey/api/attachments/Attachment.java index 904972dae6e..b946c8012d4 100644 --- a/api/src/org/labkey/api/attachments/Attachment.java +++ b/api/src/org/labkey/api/attachments/Attachment.java @@ -21,9 +21,13 @@ import org.labkey.api.security.User; import org.labkey.api.security.UserManager; import org.labkey.api.services.ServiceRegistry; +import org.labkey.api.util.DOM; +import org.labkey.api.util.HtmlString; import org.labkey.api.util.MemTracker; import org.labkey.api.util.MimeMap; +import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Path; +import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewServlet; import org.labkey.api.webdav.WebdavResolver; @@ -350,4 +354,29 @@ public void setDocumentSize(int documentSize) { _documentSize = documentSize; } + + /** + * Returns an HtmlString rendering a download link: an anchor containing a file type icon and the filename. + * The icon is marked aria-hidden since it is decorative; the link text serves as the accessible name. + */ + public HtmlString renderDownloadLink(ActionURL downloadURL) + { + return renderDownloadLink(downloadURL, getName()); + } + + /** + * Returns an HtmlString rendering a download link: an anchor containing a file type icon and custom link text. + * Use this overload when the visible link label differs from the filename (e.g. "Study Protocol Document"). + * The icon is marked aria-hidden since it is decorative; linkText serves as the accessible name. + */ + public HtmlString renderDownloadLink(ActionURL downloadURL, String linkText) + { + return DOM.createHtmlFragment( + DOM.A(DOM.at(DOM.Attribute.href, downloadURL.toString()), + DOM.IMG(DOM.at(DOM.Attribute.alt, "").at(DOM.Attribute.src, PageFlowUtil.staticResourceUrl(getFileIcon()))), + HtmlString.NBSP, + linkText + ) + ); + } } diff --git a/api/src/org/labkey/api/mcp/McpService.java b/api/src/org/labkey/api/mcp/McpService.java index 63bfcaabfa6..6ad04872537 100644 --- a/api/src/org/labkey/api/mcp/McpService.java +++ b/api/src/org/labkey/api/mcp/McpService.java @@ -29,11 +29,11 @@ /// ### MCP Development Guide /// `McpService` lets you expose functionality over the MCP protocol (only simple http for now). This allows external /// chat sessions to pull information from LabKey Server. Exposed functionality is also made available to chat sessions -/// hosted by LabKey (see `AbstractAgentAction``). +/// hosted by LabKey (see `AbstractAgentAction`). /// /// ### Adding a new MCP class /// 1. Create a new class that implements `McpImpl` (see below) in the appropriate module -/// 2. Register that class in your module `init()` method: `McpService.get().register(new MyMcp())` +/// 2. Register that class in your module's `startup()` method: `McpService.get().register(new MyMcp())` /// 3. Add tools and resources /// /// ### Adding a new MCP tool @@ -44,13 +44,13 @@ /// permission annotation is required, otherwise your tool will not be registered.** /// 4. Add `ToolContext` as the first parameter to the method /// 5. Add additional required or optional parameters to the method signature, as needed. Note that "required" is the -/// default. Again here, the parameter descriptions are very important. Provide examples. +/// default. Again here, the parameter descriptions are very important. Provide examples of parameter values. /// 6. Use the helper method `getContext(ToolContext)` to retrieve the current `Container` and `User` /// 7. Use the helper method `getUser(ToolContext)` in the rare cases where you need just a `User` /// 8. Perform additional permissions checking (beyond what the annotations offer), where appropriate /// 9. Filter all results to the current container, of course /// 10. For any error conditions, throw exceptions with detailed information. These will get translated into appropriate -/// failure responses and the LLM client will attempt to correct the problem. +/// failure responses and the LLM client will attempt to correct any problems (hopefully). /// 11. For success cases, return a String with a message or JSON content, for example, `JSONObject.toString()`. Spring /// has some limited ability to convert other objects into JSON strings, but we haven't experimented with that. See /// `DefaultToolCallResultConverter` and the ability to provide a custom result converter via the `@Tool` annotation. @@ -126,6 +126,7 @@ static void setInstance(McpService service) boolean isReady(); + // Register MCPs in Module.startup() default void register(McpImpl mcp) { try diff --git a/api/src/org/labkey/api/module/ModuleLoader.java b/api/src/org/labkey/api/module/ModuleLoader.java index a43af596db6..6248d1b0e75 100644 --- a/api/src/org/labkey/api/module/ModuleLoader.java +++ b/api/src/org/labkey/api/module/ModuleLoader.java @@ -580,19 +580,6 @@ private void doInit(Execution execution) throws ServletException throw new IllegalStateException("Core module was not first or could not find the Core module. Ensure that Tomcat user can create directories under the /modules directory."); setProjectRoot(coreModule); - for (Module module : modules) - { - module.registerFilters(_servletContext); - } - for (Module module : modules) - { - module.registerServlets(_servletContext); - } - for (Module module : modules) - { - module.registerFinalServlets(_servletContext); - } - // Do this after we've checked to see if we can find the core module. See issue 22797. verifyProductionModeMatchesBuild(); @@ -790,12 +777,34 @@ public void addStaticWarnings(@NotNull Warnings warnings, boolean showAllWarning if (!modulesRequiringUpgrade.isEmpty() || !additionalSchemasRequiringUpgrade.isEmpty()) setUpgradeState(UpgradeState.UpgradeRequired); - // Don't accept any requests if we're bootstrapping empty schemas or migrating from SQL Server + // Don't accept any requests if we're bootstrapping empty schemas or doing a database migration if (!shouldInsertData()) execution = Execution.Synchronous; startNonCoreUpgradeAndStartup(execution, lockFile); + // Register filters and servlets at the last minute, just before Tomcat starts. At this point, the list of + // modules is final. We've had one case where the CSP filter was getting initialized before the core module + // was initialized, GitHub Issue 1008. We have no idea how that happened, but registering late won't hurt. + + _log.info("Registering filters"); + + for (Module module : _modules) + { + module.registerFilters(_servletContext); + } + + _log.info("Registering servlets"); + + for (Module module : _modules) + { + module.registerServlets(_servletContext); + } + for (Module module : _modules) + { + module.registerFinalServlets(_servletContext); + } + _log.info("LabKey Server startup is complete; {}", execution.getLogMessage()); } diff --git a/api/src/org/labkey/api/study/actions/ParticipantVisitResolverChooser.java b/api/src/org/labkey/api/study/actions/ParticipantVisitResolverChooser.java index c9a6d34f1ef..d0b733ea6b6 100644 --- a/api/src/org/labkey/api/study/actions/ParticipantVisitResolverChooser.java +++ b/api/src/org/labkey/api/study/actions/ParticipantVisitResolverChooser.java @@ -168,7 +168,7 @@ public void renderInputHtml(RenderContext ctx, HtmlWriter out, Object value) return ret2; } ), - disabledInput ? InputBuilder.hidden().name(_typeInputName).value(finalSelected.getName()).appendTo(out) : null + disabledInput ? InputBuilder.hidden().name(_typeInputName).value(finalSelected.getName()).getHtmlString() : null ) ).appendTo(out); } diff --git a/api/src/org/labkey/api/util/DOM.java b/api/src/org/labkey/api/util/DOM.java index 1feb02da7b1..140e91deba9 100644 --- a/api/src/org/labkey/api/util/DOM.java +++ b/api/src/org/labkey/api/util/DOM.java @@ -362,6 +362,54 @@ public enum Attribute action, align, alt, + aria_activedescendant, + aria_atomic, + aria_autocomplete, + aria_busy, + aria_checked, + aria_colcount, + aria_colindex, + aria_colspan, + aria_controls, + aria_current, + aria_describedby, + aria_details, + aria_disabled, + aria_dropeffect, + aria_errormessage, + aria_expanded, + aria_flowto, + aria_grabbed, + aria_haspopup, + aria_hidden, + aria_invalid, + aria_keyshortcuts, + aria_label, + aria_labelledby, + aria_level, + aria_live, + aria_modal, + aria_multiline, + aria_multiselectable, + aria_orientation, + aria_owns, + aria_placeholder, + aria_posinset, + aria_pressed, + aria_readonly, + aria_relevant, + aria_required, + aria_roledescription, + aria_rowcount, + aria_rowindex, + aria_rowspan, + aria_selected, + aria_setsize, + aria_sort, + aria_valuemax, + aria_valuemin, + aria_valuenow, + aria_valuetext, async, autocomplete, autofocus, @@ -570,7 +618,6 @@ public _Attributes data(boolean condition, String datakey, Object value) } return this; } - public _Attributes cl(String...names) { if (null != names) @@ -891,7 +938,7 @@ private static Appendable appendAttribute(Appendable html, Attribute key, Object if (null==value) return html; html.append(" "); - html.append(key.name()); + html.append(key.name().replace('_', '-')); html.append("=\""); // NOTE it is somewhat unusual to pass in a Renderable, but it is possible that we // want to render HTML into an attribute. We still need to re-encode the value before trying to wrap with "". diff --git a/api/src/org/labkey/api/workflow/Action.java b/api/src/org/labkey/api/workflow/Action.java new file mode 100644 index 00000000000..204d8356115 --- /dev/null +++ b/api/src/org/labkey/api/workflow/Action.java @@ -0,0 +1,130 @@ +package org.labkey.api.workflow; + +import org.apache.commons.lang3.StringUtils; +import org.json.JSONObject; +import org.labkey.api.data.CreatedModified; +import org.labkey.api.util.GUID; + +import java.util.LinkedHashMap; +import java.util.Map; + +public abstract class Action extends CreatedModified +{ + public static final String ASSAY_TYPES_KEY = "assayTypes"; + protected Long _rowId; + protected GUID _containerId; + protected String _name; + protected boolean _isUpdatable = false; + protected Long _taskId; + protected WorkflowService.ActionType _type; + protected JSONObject _inputParameters; + + + public Long getRowId() + { + return _rowId; + } + + public void setRowId(Long rowId) + { + _rowId = rowId; + } + + public GUID getContainerId() + { + return _containerId; + } + + public void setContainerId(GUID containerId) + { + _containerId = containerId; + } + + public String getName() + { + return _name; + } + + public void setName(String name) + { + _name = name; + } + + public boolean getIsUpdatable() + { + return _isUpdatable; + } + + public void setIsUpdatable(boolean updatable) + { + _isUpdatable = updatable; + } + + public Long getTaskId() + { + return _taskId; + } + + public void setTaskId(Long taskId) + { + _taskId = taskId; + } + + public WorkflowService.ActionType getType() + { + return _type; + } + + public void setType(WorkflowService.ActionType type) + { + _type = type; + } + + public JSONObject getInputParameters() + { + return _inputParameters; + } + + public void setInputParameters(JSONObject inputParameters) + { + _inputParameters = inputParameters; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Action action = (Action) o; + + // GitHub Issue 799: Workflow Automation: Attempting to add a sample filter on an existing template errors + // Migration script generated action.name, but they are currently not used. Allow name to be changed to null or empty string. + if (!java.util.Objects.equals(_name, action._name) && !StringUtils.isEmpty(action.getName())) + return false; + + return _isUpdatable == action._isUpdatable && + java.util.Objects.equals(_rowId, action._rowId) && + java.util.Objects.equals(_taskId, action._taskId) && + java.util.Objects.equals(_type, action._type) && + java.util.Objects.equals( + _inputParameters == null ? null : _inputParameters.toString(), + action._inputParameters == null ? null : action._inputParameters.toString() + ); + } + + public Map toAuditDetailMap() + { + Map map = new LinkedHashMap<>(); + map.put("rowId", _rowId); + map.put("name", _name); + map.put("isUpdatable", _isUpdatable); + map.put("taskId", _taskId); + if (_type != null) + map.put("type", _type.name()); + if (_inputParameters != null) + map.put("inputParameters", _inputParameters.toString()); + return map; + } + + public abstract Task getTask(); +} diff --git a/api/src/org/labkey/api/workflow/Job.java b/api/src/org/labkey/api/workflow/Job.java new file mode 100644 index 00000000000..7765baf45d0 --- /dev/null +++ b/api/src/org/labkey/api/workflow/Job.java @@ -0,0 +1,365 @@ +package org.labkey.api.workflow; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.jetbrains.annotations.NotNull; +import org.json.JSONObject; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CreatedModified; +import org.labkey.api.exp.Identifiable; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.ObjectProperty; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.Group; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.util.GUID; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public abstract class Job extends CreatedModified implements Identifiable +{ + protected Long _rowId; + protected GUID _containerId; + protected Container _container; + protected GUID _entityId; + + protected String _name; + protected String _id; + protected String _description; + protected Date _startDate; + protected Date _dueDate; + protected Integer _priority; + protected Long _templateId; + protected Integer _assignee; + protected List _notifyList; + protected boolean _isTemplate; + protected Job _template; + protected Integer _jobCount; // only applies to templates + protected Integer _domainId; + protected String _lsid; // needed for attaching domain properties + protected List _tasks; // ordered by ordinal value + protected List _attachments; + protected List> _attachmentData = new ArrayList<>(); // ?? used for template instead of attachments? + protected List _entities; + protected final Map> _entityTypes = new HashMap<>(); + protected Map _domainProperties; + protected Map _domainPropertyValues; + protected boolean _hasMedia = false; + + public Long getRowId() + { + return _rowId; + } + + public void setRowId(Long rowId) + { + _rowId = rowId; + } + + public GUID getContainerId() + { + return _containerId; + } + + public void setContainerId(GUID containerId) + { + _containerId = containerId; + } + + @JsonIgnore + public Container getContainer() + { + if (_containerId == null) + return null; + if (_container == null) + _container = ContainerManager.getForId(_containerId); + return _container; + } + + public GUID getEntityId() + { + return _entityId; + } + + public void setEntityId(GUID entityId) + { + _entityId = entityId; + } + + public String getName() + { + return _name; + } + + public void setName(String name) + { + _name = name; + } + + public String getId() + { + return _id; + } + + public void setId(String id) + { + _id = id; + } + + public String getDescription() + { + return _description; + } + + public void setDescription(String description) + { + _description = description; + } + + public Date getStartDate() + { + return _startDate; + } + + public void setStartDate(Date startDate) + { + _startDate = startDate; + } + + public Date getDueDate() + { + return _dueDate; + } + + public void setDueDate(Date dueDate) + { + _dueDate = dueDate; + } + + public Integer getPriority() + { + return _priority; + } + + public void setPriority(Integer priority) + { + _priority = priority; + } + + public Long getTemplateId() + { + return _templateId; + } + + public void setTemplateId(Long templateId) + { + _templateId = templateId; + } + + @JsonProperty("template") + public abstract Job getTemplate(Container container, User user); + + public static JSONObject getAssigneeJSON(Integer assigneeId) + { + if (assigneeId == null) + return null; + User user = UserManager.getUser(assigneeId); + if (user != null) + return user.getUserProps(); + + Group group = SecurityManager.getGroup(assigneeId); + if (group != null) + { + JSONObject props = new JSONObject(); + props.put("id", group.getUserId()); + props.put("displayName", group.getName()); + return props; + } + + return null; + } + + @JsonProperty("assignee") + public void setAssignee(Integer assignee) + { + _assignee = assignee; + } + + public Integer getAssignee() + { + return _assignee; + } + + @JsonProperty("assignee") + public JSONObject getAssigneeJSON() + { + return getAssigneeJSON(_assignee); + } + + public abstract List getNotifyList(); + + public void setNotifyList(List notifyList) + { + _notifyList = notifyList; + } + + @JsonProperty("notifyList") + public List getNotifyListJSON() + { + if (getNotifyList() == null) + return null; + + return getNotifyList().stream() + .map(Job::getAssigneeJSON) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + public boolean getIsTemplate() + { + return _isTemplate; + } + + public void setIsTemplate(boolean template) + { + _isTemplate = template; + } + + public Integer getJobCount() + { + return _jobCount; + } + + public void setJobCount(Integer jobCount) + { + _jobCount = jobCount; + } + + @Override + public String getLSID() + { + return _lsid; + } + + public void setLSID(String lsid) + { + _lsid = lsid; + } + + public void setLSID(Lsid lsid) + { + _lsid = lsid.toString(); + } + + public Integer getDomainId() + { + return _domainId; + } + + public void setDomainId(Integer domainId) + { + _domainId = domainId; + } + + public abstract List getTasks(); + + public void setTasks(List tasks) + { + _tasks = tasks; + } + + @JsonIgnore + public List getAttachments() + { + return _attachments; + } + + public void setAttachments(List attachments) + { + _attachments = attachments; + } + + public abstract List getEntities(); + + @JsonIgnore + public abstract @NotNull List getSamples(); + + public void setEntities(List entities) + { + _entities = entities; + } + + @JsonProperty("containerPath") + public String getContainerPath() + { + Container container = getContainer(); + return container == null ? null : container.getPath(); + } + + @JsonProperty("hasMedia") + public boolean isHasMedia() + { + return _hasMedia; + } + + public void setHasMedia(boolean hasMedia) + { + _hasMedia = hasMedia; + } + + public abstract Map toMap(); + + @JsonIgnore + public List getSubsequentTasks(long taskId) + { + List orderedTasks = getTasks().stream().sorted().toList(); + int index = orderedTasks.stream().map(Task::getRowId).toList().indexOf(taskId); + if (index == -1 || index == orderedTasks.size() - 1) + return Collections.emptyList(); + return orderedTasks.subList(index+1, orderedTasks.size()); + } + + public Task getNextTask(long taskId) + { + List subsequent = getSubsequentTasks(taskId); + if (subsequent.isEmpty()) + return null; + return subsequent.get(0); + } + + public boolean isComplete() + { + if (getTasks().isEmpty()) + return true; + return getTasks().stream().allMatch(Task::isCompleted); + } + + public List> getAttachmentData() + { + return _attachmentData; + } + + public void setAttachmentData(List> attachmentData) + { + _attachmentData = attachmentData; + } + + public abstract Map toAuditDetailMap(); + + public abstract Object getDomainProperty(PropertyDescriptor prop); + + public abstract void setProperty(User user, PropertyDescriptor pd, Object value) throws ValidationException; +} diff --git a/api/src/org/labkey/api/workflow/Task.java b/api/src/org/labkey/api/workflow/Task.java new file mode 100644 index 00000000000..627fec251f0 --- /dev/null +++ b/api/src/org/labkey/api/workflow/Task.java @@ -0,0 +1,274 @@ +package org.labkey.api.workflow; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.jetbrains.annotations.NotNull; +import org.json.JSONObject; +import org.labkey.api.data.Container; +import org.labkey.api.data.CreatedModified; +import org.labkey.api.util.GUID; + +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +@JsonIgnoreProperties(ignoreUnknown = true) +public abstract class Task extends CreatedModified implements Comparable +{ + protected Long _rowId; + protected GUID _containerId; + protected Container _container; + protected GUID _entityId; + + protected String _name; + + protected String _description; + protected String _entityFilter; + protected Integer _status; + protected Date _startDate; + protected Date _endDate; + protected Date _dueDate; + protected int _ordinal; + protected Integer _assignee; + protected Long _jobId; + protected Job _job; + protected List _actions = null; + + public Long getRowId() + { + return _rowId; + } + + public void setRowId(Long rowId) + { + _rowId = rowId; + } + + public String getName() + { + return _name; + } + + public void setName(String name) + { + _name = name; + } + + public String getDescription() + { + return _description; + } + + public void setDescription(String description) + { + _description = description; + } + + public String getEntityFilter() + { + return _entityFilter; + } + + public void setEntityFilter(String entityFilter) + { + _entityFilter = entityFilter; + } + + @JsonProperty("assignee") + public JSONObject getAssigneeJSON() + { + return Job.getAssigneeJSON(_assignee); + } + + public Integer getAssignee() + { + return _assignee; + } + + @JsonProperty("assignee") + public void setAssignee(Integer assignee) + { + _assignee = assignee; + } + + public Integer getStatus() + { + return _status; + } + + public void setStatus(Integer status) + { + _status = status; + } + + public int getOrdinal() + { + return _ordinal; + } + + public void setOrdinal(int ordinal) + { + _ordinal = ordinal; + } + + public Date getStartDate() + { + return _startDate; + } + + public void setStartDate(Date startDate) + { + _startDate = startDate; + } + + public Date getEndDate() + { + return _endDate; + } + + public void setEndDate(Date endDate) + { + _endDate = endDate; + } + + public Date getDueDate() + { + return _dueDate; + } + + public void setDueDate(Date dueDate) + { + _dueDate = dueDate; + } + + public GUID getEntityId() + { + return _entityId; + } + + public void setEntityId(GUID entityId) + { + _entityId = entityId; + } + + public GUID getContainerId() + { + return _containerId; + } + + public void setContainerId(GUID containerId) + { + _containerId = containerId; + } + + @JsonIgnore + public Container getContainer() + { + return _container; + } + + public void setContainer(Container container) + { + _container = container; + } + + public abstract boolean isCompleted(); + + public abstract boolean isActive(); + + public abstract boolean isPending(); + + public abstract List getActions(); + + public void setActions(List actions) + { + _actions = actions; + } + + public Long getJobId() + { + return _jobId; + } + + public void setJobId(Long jobId) + { + _jobId = jobId; + } + + public abstract Job getJob(); + + public abstract Map toMap(); + + // Determine if the template task with existing jobs can be updated to the new task definition + // Only entityFilter field is allowed to be changed for a referenced template task + public boolean canUpdateUsedTemplateTask(Task task) + { + if (this == task) return true; + if (task == null || getClass() != task.getClass()) return false; + + if (!Objects.equals(_name, task._name) || + !Objects.equals(_description, task._description)) + return false; + + List existingActions = this.getActions(); + List newActions = task.getActions(); + if (existingActions.size() != newActions.size()) + return false; + + for (int i = 0; i < existingActions.size(); i++) + { + Action existingAction = existingActions.get(i); + Action newAction = newActions.get(i); + if (!existingAction.equals(newAction)) + { + return false; + } + } + + return true; + } + + @JsonIgnore + public boolean hasUpdatableAssaysAction() + { + return getActions().stream().anyMatch(action -> action.getType() == WorkflowService.ActionType.AssayImport && action.getIsUpdatable()); + } + + public Map toAuditDetailMap() + { + Map map = new LinkedHashMap<>(); + map.put("rowId", getRowId()); + map.put("name", getName()); + map.put("description", getDescription()); + map.put("entityFilter", getEntityFilter()); + map.put("status", getStatus()); + if (getStartDate() != null) + map.put("startDate", getStartDate()); + if (getEndDate() != null) + map.put("endDate", getEndDate()); + if (getDueDate() != null) + map.put("dueDate", getDueDate()); + if (getEntityId() != null) + map.put("entityId", getEntityId().toString()); + map.put("ordinal", getOrdinal()); + int actionIndex = 1; + for (Action action : getActions()) + { + Map actionMap = action.toAuditDetailMap(); + for (Map.Entry entry : actionMap.entrySet()) + map.put("action" + actionIndex + "." + entry.getKey(), entry.getValue()); + actionIndex++; + } + return map; + } + + @Override + public int compareTo(@NotNull Task o) + { + return Integer.compare(getOrdinal(), o.getOrdinal()); + } + +} diff --git a/api/src/org/labkey/api/workflow/WorkEntity.java b/api/src/org/labkey/api/workflow/WorkEntity.java new file mode 100644 index 00000000000..629ce889eb9 --- /dev/null +++ b/api/src/org/labkey/api/workflow/WorkEntity.java @@ -0,0 +1,199 @@ +package org.labkey.api.workflow; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.json.JSONObject; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.util.GUID; + +import java.util.Date; +import java.util.Map; + +public class WorkEntity +{ + public enum WorkType + { + Job, + Task, + Action + } + + public enum EntityType + { + Sample, + Source, + Plate, + } + + protected Long _rowId; + protected GUID _containerId; + protected WorkType _workType; + protected Long _workRowId; + protected EntityType _entityType; + protected Long _entityValue; + protected Long _created; + protected User _createdBy; + + public WorkEntity() + {} + + public WorkEntity(Map map) + { + _rowId = MapUtils.getLong(map, "rowId"); + if (map.get("workType") instanceof String) + _workType = WorkType.valueOf((String) map.get("workType")); + else + _workType = (WorkType) map.get("workType"); + _workRowId = MapUtils.getLong(map, "workRowId"); + if (map.get("entityType") instanceof String) + _entityType = EntityType.valueOf((String) map.get("entityType")); + else + _entityType = (EntityType) map.get("entityType"); + _entityValue = MapUtils.getLong(map, "entityValue"); + if (map.get("Container") != null) + this.setContainerId(new GUID((String) map.get("Container"))); + } + + public WorkEntity(ExpMaterial sample) + { + _entityType = EntityType.Sample; + _entityValue = sample.getRowId(); + } + + public WorkEntity(ExpMaterial sample, WorkType workType, Long workRowId) + { + this(sample); + _workType = workType; + _workRowId = workRowId; + } + + + public Long getRowId() + { + return _rowId; + } + + public void setRowId(Long rowId) + { + _rowId = rowId; + } + + public GUID getContainerId() + { + return _containerId; + } + + public void setContainerId(GUID containerId) + { + _containerId = containerId; + } + + @JsonProperty("created") + public Long getCreated() + { + return _created; + } + + @JsonIgnore + public Date getCreatedDate() + { + return _created == null ? null : new Date(_created); + } + + public void setCreated(Long created) + { + _created = created; + } + + @JsonIgnore // created is serialized as Long + public void setCreated(Date created) + { + if (created != null) + setCreated(created.getTime()); + } + + @JsonProperty("createdBy") + public JSONObject getCreatedBy() + { + if (_createdBy == null) + return null; + return _createdBy.getUserProps(); + } + + @JsonIgnore + public User getCreatedByUser() + { + return _createdBy; + } + + public void setCreatedBy(User createdBy) + { + _createdBy = createdBy; + } + + public void setCreatedBy(Integer createdById) + { + if (createdById != null) + _createdBy = UserManager.getUser(createdById); + } + + public WorkType getWorkType() + { + return _workType; + } + + public void setWorkType(WorkType workType) + { + _workType = workType; + } + + public void setWorkType(String workTypeStr) + { + _workType = StringUtils.isEmpty(StringUtils.trimToEmpty(workTypeStr)) ? null : WorkType.valueOf(workTypeStr.trim()); + } + + public Long getWorkRowId() + { + return _workRowId; + } + + public void setWorkRowId(Long workRowId) + { + _workRowId = workRowId; + } + + public EntityType getEntityType() + { + return _entityType; + } + + public void setEntityType(EntityType entityType) + { + _entityType = entityType; + } + + public void setValueType(String valueTypeStr) + { + _entityType = EntityType.valueOf(valueTypeStr); + } + + public Long getEntityValue() + { + return _entityValue; + } + + public void setEntityValue(Long entityValue) + { + _entityValue = entityValue; + } + + @JsonIgnore + public String getKey() + { + return _entityType + ":" + _entityValue; + } +} diff --git a/api/src/org/labkey/api/workflow/WorkflowService.java b/api/src/org/labkey/api/workflow/WorkflowService.java index 55e22cb6602..6c830e01625 100644 --- a/api/src/org/labkey/api/workflow/WorkflowService.java +++ b/api/src/org/labkey/api/workflow/WorkflowService.java @@ -1,7 +1,10 @@ package org.labkey.api.workflow; import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.Container;import org.labkey.api.security.User;import org.labkey.api.services.ServiceRegistry; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.security.User; +import org.labkey.api.services.ServiceRegistry; public interface WorkflowService { @@ -41,4 +44,10 @@ static WorkflowService get() void onActionComplete(@NotNull Container container, @NotNull User user, @NotNull Long actionId); void onActionComplete(@NotNull Container container, @NotNull User user, @NotNull Long taskId, @NotNull ActionType actionType); + + @Nullable + Job getJob(Long jobId); + + @Nullable + Job getELNReferencePlaceholderJob(Container container); } diff --git a/api/src/org/labkey/filters/ContentSecurityPolicyFilter.java b/api/src/org/labkey/filters/ContentSecurityPolicyFilter.java index 90a9b1c5437..dc077525b6b 100644 --- a/api/src/org/labkey/filters/ContentSecurityPolicyFilter.java +++ b/api/src/org/labkey/filters/ContentSecurityPolicyFilter.java @@ -12,7 +12,6 @@ import org.apache.commons.collections4.multimap.HashSetValuedHashMap; import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -30,7 +29,6 @@ import org.labkey.api.util.StringExpressionFactory; import org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior; import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; import java.io.IOException; import java.security.SecureRandom; @@ -40,13 +38,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; - /** * Content Security Policies (CSPs) are loaded from the csp.enforce and csp.report properties in application.properties. */ @@ -57,6 +53,7 @@ public class ContentSecurityPolicyFilter implements Filter private static final String NONCE_SUBST = "REQUEST.SCRIPT.NONCE"; private static final String UPGRADE_INSECURE_REQUESTS_SUBSTITUTION = "UPGRADE.INSECURE.REQUESTS"; private static final String HEADER_NONCE = "org.labkey.filters.ContentSecurityPolicyFilter#NONCE"; // needs to match PageConfig.HEADER_NONCE + private static final String REPORTING_ENDPOINTS_HEADER = "Reporting-Endpoints"; private static final Map CSP_FILTERS = new CopyOnWriteHashMap<>(); @@ -76,11 +73,14 @@ public class ContentSecurityPolicyFilter implements Filter private @NotNull String _cspVersion = "Unknown"; // These two are effectively @NotNull since they are set to non-null values in init() and never changed private String _stashedTemplate = null; - private String _reportToEndpointName = null; - // Per-filter-instance settings are initialized on first request and reset when base server URL or allowed sources - // change. Don't reference this directly; always use ensureSettings(). - private volatile @Nullable CspFilterSettings _settings = null; + // We can't set this statically because the class is referenced before URLProviders are available + @SuppressWarnings("DataFlowIssue") + private final String _reportingEndpointsHeaderValue = "csp-report=\"" + PageFlowUtil.urlProvider(AdminUrls.class).getCspReportToURL().getLocalURIString() + "\""; + + // Initialized on first request and reset when allowed sources change. Don't reference this directly; always use + // ensurePolicyExpression(). + private volatile StringExpression _policyExpression; public enum ContentSecurityPolicyType { @@ -118,7 +118,7 @@ public String getHeaderName() @Override public void init(FilterConfig filterConfig) throws ServletException { - LogHelper.getLogger(ContentSecurityPolicyFilter.class, "CSP filter initialization").info("Initializing {}", filterConfig.getFilterName()); + LOG.info("Initializing {}", filterConfig.getFilterName()); Enumeration paramNames = filterConfig.getInitParameterNames(); while (paramNames.hasMoreElements()) { @@ -146,9 +146,6 @@ else if ("disposition".equalsIgnoreCase(paramName)) if (CSP_FILTERS.put(getType(), this) != null) throw new ServletException("ContentSecurityPolicyFilter is misconfigured, duplicate policies of type: " + getType()); - - // configure a different endpoint for each type. TODO: We only need one CSP violation reporting endpoint now, so one header would do - _reportToEndpointName = "csp-" + getType().name().toLowerCase(); } /** Filter out block comments and replace special characters in the provided policy */ @@ -197,18 +194,21 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha { if (request instanceof HttpServletRequest req && response instanceof HttpServletResponse resp) { - CspFilterSettings settings = ensureSettings(); + StringExpression expression = ensurePolicyExpression(); if (getType() != ContentSecurityPolicyType.Enforce || !OptionalFeatureService.get().isFeatureEnabled(FEATURE_FLAG_DISABLE_ENFORCE_CSP)) { Map map = Map.of(NONCE_SUBST, getScriptNonceHeader(req)); - var csp = settings.getPolicyExpression().eval(map); - resp.setHeader(getType().getHeaderName(), csp); + String csp = expression.eval(map); + + if ("https".equals(req.getScheme())) + { + if (resp.getHeader(REPORTING_ENDPOINTS_HEADER) == null) + resp.addHeader(REPORTING_ENDPOINTS_HEADER, _reportingEndpointsHeaderValue); + csp = csp + " report-to csp-report ;"; + } - // null if https: is not configured on this server - String reportingEndpointsHeaderValue = settings.getReportingEndpointsHeaderValue(); - if (reportingEndpointsHeaderValue != null) - resp.addHeader("Reporting-Endpoints", reportingEndpointsHeaderValue); + resp.setHeader(getType().getHeaderName(), csp); } } chain.doFilter(request, response); @@ -229,91 +229,26 @@ public String getStashedTemplate() return _stashedTemplate; } - public String getReportToEndpointName() + private void clearPolicyExpression() { - return _reportToEndpointName; + _policyExpression = null; } - private void clearSettings() + private @NotNull StringExpression ensurePolicyExpression() { - _settings = null; - } + StringExpression expression = _policyExpression; - private @NotNull CspFilterSettings ensureSettings() - { - String baseServerUrl = AppProps.getInstance().getBaseServerUrl(); - CspFilterSettings settings = _settings; // Stash a local copy to ensure consistency in the checks below - - // Reset settings if null or if base server URL has changed - if (null == settings || !Objects.equals(baseServerUrl, settings.getPreviousBaseServerUrl())) + if (expression == null) { - settings = _settings = new CspFilterSettings(this, baseServerUrl); - } - - return settings; - } - - // Hold all the mutable per-filter settings in a single object so they can be set atomically - private static class CspFilterSettings - { - private final String _policyTemplate; - private final String _reportingEndpointsHeaderValue; - private final String _previousBaseServerUrl; - private final StringExpression _policyExpression; - - private CspFilterSettings(ContentSecurityPolicyFilter filter, String baseServerUrl) - { - // Add "Reporting-Endpoints" header and "report-to" directive only if https: is configured on this - // server. This ensures that browsers fall-back on report-uri if https: isn't configured. - if (Strings.CI.startsWith(baseServerUrl, "https://")) - { - // Each filter adds its own "Reporting-Endpoints" header since we want to convey the correct version (eXX vs. rXX) - @SuppressWarnings("DataFlowIssue") - ActionURL violationUrl = PageFlowUtil.urlProvider(AdminUrls.class).getCspReportToURL(); - // Use an absolute URL so we always post to https:, even if the violating request uses http: - _reportingEndpointsHeaderValue = filter.getReportToEndpointName() + "=\"" + violationUrl.getURIString() + "\""; - - // Add "report-to" directive to the policy - _policyTemplate = filter.getStashedTemplate() + " report-to " + filter.getReportToEndpointName() + " ;"; - } - else - { - _policyTemplate = filter.getStashedTemplate(); - _reportingEndpointsHeaderValue = null; - } - - _previousBaseServerUrl = baseServerUrl; - - final String substitutedPolicy; - synchronized (SUBSTITUTION_LOCK) { - substitutedPolicy = StringExpressionFactory.create(_policyTemplate, false, NullValueBehavior.KeepSubstitution) + var substitutedPolicy = StringExpressionFactory.create(getStashedTemplate(), false, NullValueBehavior.KeepSubstitution) .eval(SUBSTITUTION_MAP); + expression = _policyExpression = StringExpressionFactory.create(substitutedPolicy, false, NullValueBehavior.ReplaceNullAndMissingWithBlank); } - - _policyExpression = StringExpressionFactory.create(substitutedPolicy, false, NullValueBehavior.ReplaceNullAndMissingWithBlank); } - public String getPolicyTemplate() - { - return _policyTemplate; - } - - public String getReportingEndpointsHeaderValue() - { - return _reportingEndpointsHeaderValue; - } - - public String getPreviousBaseServerUrl() - { - return _previousBaseServerUrl; - } - - public StringExpression getPolicyExpression() - { - return _policyExpression; - } + return expression; } public static String getScriptNonceHeader(HttpServletRequest request) @@ -362,7 +297,7 @@ public static void unregisterAllowedSources(String key, Directive directive) /** * Regenerate the substitution map on every register/unregister. The policy expression will be regenerated on the - * next request (see {@link #ensureSettings()}). + * next request (see {@link #ensurePolicyExpression()}). */ public static void regenerateSubstitutionMap() { @@ -389,7 +324,7 @@ public static void regenerateSubstitutionMap() // Tell each registered ContentSecurityPolicyFilter to clear its settings so the next request recreates them // using the new substitution map - CSP_FILTERS.values().forEach(ContentSecurityPolicyFilter::clearSettings); + CSP_FILTERS.values().forEach(ContentSecurityPolicyFilter::clearPolicyExpression); } } @@ -436,7 +371,7 @@ public static List getMissingSubstitutions(ContentSecurityPolicyType typ } else { - String template = filter.ensureSettings().getPolicyTemplate(); + String template = filter.getStashedTemplate(); ret = Arrays.stream(Directive.values()) .map(dir -> "${" + dir.getSubstitutionKey() + "}") .filter(key -> !template.contains(key)) @@ -451,13 +386,15 @@ public static void registerMetricsProvider() UsageMetricsService.get().registerUsageMetrics("API", () -> Map.of("cspFilters", CSP_FILTERS.values().stream() .collect(Collectors.toMap(ContentSecurityPolicyFilter::getType, filter -> { - CspFilterSettings settings = filter.ensureSettings(); + StringExpression expression = filter.ensurePolicyExpression(); return Map.of( "version", filter.getCspVersion(), - "csp", settings.getPolicyTemplate(), - "cspSubstituted", settings.getPolicyExpression().getSource() + "csp", filter.getStashedTemplate(), + "cspSubstituted", expression.getSource() ); - })))); + })) + ) + ); } public static class TestCase extends Assert @@ -630,7 +567,7 @@ public void testSubstitutionMap() private void verifySubstitutionInPolicyExpressions(String value, int expectedCount) { List failures = CSP_FILTERS.values().stream() - .map(filter -> filter.ensureSettings().getPolicyExpression().eval(Map.of())) + .map(filter -> filter.ensurePolicyExpression().eval(Map.of())) .filter(policy -> StringUtils.countMatches(policy, value) != expectedCount) .toList(); diff --git a/api/webapp/clientapi/dom/DataRegion.js b/api/webapp/clientapi/dom/DataRegion.js index 07ddf75f5cc..a13e1f2c335 100644 --- a/api/webapp/clientapi/dom/DataRegion.js +++ b/api/webapp/clientapi/dom/DataRegion.js @@ -1713,8 +1713,8 @@ if (!LABKEY.DataRegions) { ct.append([ '
', - '', - '', + '', + '', '
' ].join('')); diff --git a/core/src/org/labkey/core/CoreMcp.java b/core/src/org/labkey/core/CoreMcp.java index da97a986ba9..fa18a2a55c1 100644 --- a/core/src/org/labkey/core/CoreMcp.java +++ b/core/src/org/labkey/core/CoreMcp.java @@ -18,6 +18,7 @@ import org.labkey.api.study.Study; import org.labkey.api.study.StudyService; import org.labkey.api.util.HtmlString; +import org.labkey.api.view.ActionURL; import org.springframework.ai.chat.model.ToolContext; import org.springframework.ai.mcp.annotation.McpResource; import org.springframework.ai.tool.annotation.Tool; @@ -92,7 +93,9 @@ String listContainers(ToolContext toolContext) @Tool(description = "Every tool in this MCP requires a container path, e.g. MyProject/MyFolder. A container is also called a folder or project. " + "Please prompt the user for a container path and use this tool to save the path for this MCP session. The user can also change the container " + "during the session using this tool. The user must have read permissions in the container, in other words, the path must be on the list that " + - "the listContainers tool returns. Don't suggest a leading slash on the path because typing a slash in some LLM clients triggers custom shortcuts.") + "the listContainers tool returns. Don't suggest a leading slash on the path because typing a slash in some LLM clients triggers custom shortcuts. " + + "Alternately, the user may provide a LabKey server URL like http://localhost:8080/StudyVerifyProject/My%20Study/project-begin.view. " + + "The container path is encoded in the URL and can be accepted as a valid parameter.") @RequiresNoPermission // Because we don't have a container yet, but the tool will verify read permission before setting the container String setContainer(ToolContext context, @ToolParam(description = "Container path, e.g. MyProject/MyFolder") String containerPath) { @@ -100,6 +103,18 @@ String setContainer(ToolContext context, @ToolParam(description = "Container pat Container container = ContainerManager.getForPath(containerPath); + if (null == container && (containerPath.startsWith("http://") || containerPath.startsWith("https://"))) + { + try + { + var url = new ActionURL(containerPath); + container = ContainerManager.getForURL(url); + } + catch (IllegalArgumentException _) + { + } + } + // Must exist and user must have read permission to set a container. Note: Send the same message in either // case to prevent information exposure. if (container == null || !container.hasPermission(getUser(context), ReadPermission.class)) @@ -132,4 +147,41 @@ public ReadResourceResult getFileBasedModuleDevelopmentGuide() throws IOExceptio ) )); } + + @McpResource( + uri = "resource://org/labkey/core/DataAnalysis_Python.md", + mimeType = "application/markdown", + name = "Python Data Analysis Development Guide", + description = "Provide documentation for developers using Python to analyze LabKey data") + public ReadResourceResult getPythonDataAnalysisGuide() throws IOException + { + incrementResourceRequestCount("Python Data Analysis"); + String markdown = IOUtils.resourceToString("org/labkey/core/DataAnalysis_Python.md", null, CoreModule.class.getClassLoader()); + return new ReadResourceResult(List.of( + new McpSchema.TextResourceContents( + "resource://org/labkey/core/DataAnalysis_Python.md", + "application/markdown", + markdown + ) + )); + } + + @McpResource( + uri = "resource://org/labkey/core/DataAnalysis_R.md", + mimeType = "application/markdown", + name = "R Data Analysis Development Guide", + description = "Provide documentation for developers using R to analyze LabKey data") + public ReadResourceResult getRDataAnalysisGuide() throws IOException + { + incrementResourceRequestCount("R Data Analysis"); + String markdown = IOUtils.resourceToString("org/labkey/core/DataAnalysis_R.md", null, CoreModule.class.getClassLoader()); + return new ReadResourceResult(List.of( + new McpSchema.TextResourceContents( + "resource://org/labkey/core/DataAnalysis_R.md", + "application/markdown", + markdown + ) + )); + } + } diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index 2312ec48f5f..fd2fd568258 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -559,7 +559,6 @@ public QuerySchema createSchema(DefaultSchema schema, Module module) ScriptEngineManagerImpl.registerEncryptionMigrationHandler(); - McpService.get().register(new CoreMcp()); PostgreSqlService.setInstance(PostgreSqlDialectFactory::getLatestSupportedDialect); deleteTempFiles(); @@ -1283,6 +1282,8 @@ public void moduleStartupComplete(ServletContext servletContext) UserManager.addUserListener(new EmailPreferenceUserListener()); Encryption.checkMigration(); + + McpService.get().register(new CoreMcp()); } // Issue 7527: Auto-detect missing SQL views and attempt to recreate diff --git a/core/src/org/labkey/core/DataAnalysis_Python.md b/core/src/org/labkey/core/DataAnalysis_Python.md new file mode 100644 index 00000000000..4a83b50f278 --- /dev/null +++ b/core/src/org/labkey/core/DataAnalysis_Python.md @@ -0,0 +1,327 @@ +# LabKey Data Analysis using Python + +This project is for performing data analysis against a LabKey Server instance using AI assistance. + +**Connection defaults:** The LabKey server URL and API key can be inferred from `.mcp.json` in this directory. The `url` field (minus the `/mcp` path) provides the server endpoint, and the `apikey` header value provides the authentication token. + +**Do not embed API keys in generated scripts.** Instead, ensure a `.netrc` file (Linux/Mac) or `_netrc` file (Windows) exists in the user's home directory with the server credentials. If the file does not exist, offer to create it using the API key from `.mcp.json`. The format is: + +``` +machine +login apikey +password +``` + +The `machine` value is the hostname only — no protocol (`https://`), no port, no path. For example, for `https://myserver.labkey.com:8443/labkey/mcp`, use `myserver.labkey.com`. On Linux/Mac, set permissions to 600 (`chmod 600 ~/.netrc`). + +When writing Python scripts, read the server URL from `.mcp.json` to pre-populate the `APIWrapper` connection parameters, but omit `api_key` — the `labkey` package reads `.netrc` automatically. Before running any script, confirm with the data analyst: +- Is this the correct server? +- Should `use_ssl` be True or False? (infer from the URL scheme in `.mcp.json`) +- What container path should be used? (use MCP `listContainers` to show available options) + +Confirm all of these settings with the analyst before writing any script. + +## Online Reference Material +https://www.labkey.org/Documentation/wiki-page.view?name=python + +## MCP Tools Available + +A LabKey MCP server is configured (see `.mcp.json`). Use these tools to explore the server interactively: + +| Tool | Purpose | +|---|---| +| `mcp__labkey__setContainer` | **Must be called first.** Sets the active container (project/folder) for subsequent calls. Path format: `MyProject/MyFolder` (no leading slash). | +| `mcp__labkey__whereAmIWhoAmITalkingTo` | Shows current user, server info, and active container. | +| `mcp__labkey__listContainers` | Lists all containers the user has read access to. | +| `mcp__labkey__listSchemas` | Lists all schemas in the active container. | +| `mcp__labkey__listTables` | Lists tables/queries within a schema. | +| `mcp__labkey__listColumns` | Shows column metadata (name, type, description) for a table. Also returns SQL source for saved queries. | +| `mcp__labkey__getSourceForSavedQuery` | Returns the SQL source of a saved query. | +| `mcp__labkey__validateSQL` | Validates LabKey SQL syntax without executing it. | + +### MCP Workflow + +1. Call `listContainers` to find available containers +2. Call `setContainer` with the desired container path +3. Use `listSchemas` -> `listTables` -> `listColumns` to explore the data model +4. Use `validateSQL` to check queries before running them +5. To actually retrieve data, write a Python script using the `labkey` Python API (see below) + +## LabKey Python API (`labkey` package) + +**Install:** `pip install labkey` +**Requires:** Python 3.11+, LabKey Server v15.1+ +**Repo:** https://github.com/LabKey/labkey-api-python + +### Connection Setup + +**Preferred: `.netrc` authentication (no credentials in scripts)** + +```python +from labkey.api_wrapper import APIWrapper + +api = APIWrapper( + "localhost:8080", # domain (hostname or hostname:port) + "MyProject/MyFolder", # container path + context_path=None, # URL path segment after domain, e.g. "labkey" + use_ssl=False, # True for https + verify_ssl=True, # False for self-signed dev certs +) +``` + +The `labkey` package automatically reads credentials from `~/.netrc` (Linux/Mac) or `~/_netrc` (Windows). The `.netrc` entry should use `apikey` as the login and the API key as the password: + +``` +machine localhost +login apikey +password TheUniqueAPIKeyGeneratedForYou +``` + +The `machine` value must be the hostname only — no protocol, no port, no path. Set file permissions to 600 on Linux/Mac (`chmod 600 ~/.netrc`). + +Authentication options (in order of preference): +- **.netrc file** (recommended): Create `~/.netrc` with `machine`, `login apikey`, `password ` fields (chmod 600). No credentials appear in scripts. +- **API key in code** (avoid in generated scripts): Pass `api_key=` to APIWrapper. Only use this for quick interactive testing, not in saved scripts. + +### Data Retrieval APIs + +#### select_rows -- Query a table/view + +```python +result = api.query.select_rows( + schema_name, # str: e.g. "lists", "core", "study" + query_name, # str: table or query name + view_name=None, # str: named view to use + filter_array=None, # list[QueryFilter]: row filters + columns=None, # str: comma-separated column names + max_rows=-1, # int: -1 = ALL rows (default!) + sort=None, # str: comma-separated, prefix '-' for desc + offset=None, # int: rows to skip (pagination) + container_path=None, # str: override container + container_filter=None, # str: e.g. "CurrentAndSubfolders" + parameters=None, # dict: for parameterized queries + include_total_count=None, # bool: include total in response + timeout=300, # int: seconds +) +``` + +#### execute_sql -- Run LabKey SQL + +```python +result = api.query.execute_sql( + schema_name, # str: target schema + sql, # str: LabKey SQL + max_rows=None, # int: row limit + sort=None, # str: sort columns + offset=None, # int: row offset + container_path=None, # str: override container + container_filter=None, # str: container scope + parameters=None, # dict: query parameters + timeout=300, # int: seconds + waf_encode_sql=True, # bool: WAF encoding (needs Server v23.09+) +) +``` + +Set `waf_encode_sql=False` if targeting LabKey Server older than v23.09. + +#### get_queries -- List available tables/queries + +```python +result = api.query.get_queries( + schema_name, # str: schema to explore + container_path=None, # str: override container + include_columns=None, # bool: include column metadata + include_system_queries=None, # bool: include system-generated queries + include_title=None, # bool: include custom display titles + include_user_queries=None, # bool: include user-defined saved queries + include_view_data_url=None, # bool: include URLs for viewing data in browser + query_detail_columns=None, # bool: include detailed column info + timeout=300, # int: seconds +) +``` + +### Response Format + +Both `select_rows` and `execute_sql` return a dict: + +```python +{ + "schemaName": "lists", + "queryName": "MyTable", + "rowCount": 25, # total rows (if include_total_count=True) + "rows": [ # list of row dicts + {"Col1": "value", "Col2": 42, ...}, + ... + ], + "metaData": { + "id": "Key", # primary key column + "fields": [{"name": "Col1", "type": "string"}, ...] + } +} +``` + +Working with results: +```python +result = api.query.select_rows("lists", "MyTable") +for row in result["rows"]: + print(row["Name"], row["Value"]) + +# Convert to pandas DataFrame: +import pandas as pd +df = pd.DataFrame(result["rows"]) + +# Date columns come back as strings -- convert explicitly: +df["StartDate"] = pd.to_datetime(df["StartDate"]) + +# Numeric columns may contain None for missing values -- pandas handles this +# but be aware when doing arithmetic: +df["Age"] = pd.to_numeric(df["Age"], errors="coerce") + +# Lookup columns may return nested dicts (e.g. {"value": 1, "displayValue": "Group A"}). +# Extract the display value if needed: +if isinstance(df["Group"].iloc[0], dict): + df["Group"] = df["Group"].apply(lambda x: x.get("displayValue") if isinstance(x, dict) else x) +``` + +### Query Filters + +```python +from labkey.query import QueryFilter + +filters = [ + QueryFilter("Country", "Germany"), # equals (default) + QueryFilter("Age", "18,65", QueryFilter.Types.BETWEEN), # comma-delimited + QueryFilter("Status", "Active;Enrolled", QueryFilter.Types.IN), # semicolon-delimited + QueryFilter("Name", "", QueryFilter.Types.IS_NOT_BLANK), # no value needed +] +result = api.query.select_rows("study", "Demographics", filter_array=filters) +``` + +#### Filter Types Reference + +**Comparison:** `EQUAL`, `NOT_EQUAL` (alias `NEQ`), `GT` / `GREATER_THAN`, `LT` / `LESS_THAN`, `GTE` / `GREATER_THAN_OR_EQUAL`, `LTE` / `LESS_THAN_OR_EQUAL`, `NEQ_OR_NULL` / `NOT_EQUAL_OR_MISSING` + +**Date comparison:** `DATE_EQUAL`, `DATE_NOT_EQUAL`, `DATE_GREATER_THAN`, `DATE_LESS_THAN`, `DATE_GREATER_THAN_OR_EQUAL`, `DATE_LESS_THAN_OR_EQUAL` + +**String:** `STARTS_WITH`, `DOES_NOT_START_WITH`, `CONTAINS`, `DOES_NOT_CONTAIN`, `CONTAINS_ONE_OF`, `CONTAINS_NONE_OF` + +**Set/Range:** `IN` / `EQUALS_ONE_OF` (semicolons), `NOT_IN` / `EQUALS_NONE_OF` (semicolons), `BETWEEN` (commas), `NOT_BETWEEN` (commas) + +**Null checks (no value needed):** `IS_BLANK`, `IS_NOT_BLANK`, `HAS_MISSING_VALUE`, `DOES_NOT_HAVE_MISSING_VALUE`, `HAS_ANY_VALUE` + +**Array:** `ARRAY_CONTAINS_ALL`, `ARRAY_CONTAINS_ANY`, `ARRAY_CONTAINS_NONE`, `ARRAY_CONTAINS_EXACT`, `ARRAY_CONTAINS_NOT_EXACT`, `ARRAY_ISEMPTY`, `ARRAY_ISNOTEMPTY` + +**Search:** `Q` (full-text search across table) + +**Lineage:** `EXP_CHILD_OF`, `EXP_PARENT_OF`, `EXP_LINEAGE_OF` + +**Ontology:** `ONTOLOGY_IN_SUBTREE`, `ONTOLOGY_NOT_IN_SUBTREE` + +### Container Filters + +Control which containers are searched. Pass as `container_filter=` to select_rows/execute_sql: + +- `"Current"` -- only the active container +- `"CurrentAndSubfolders"` -- active container and its children +- `"CurrentPlusProject"` -- active container and its parent project +- `"CurrentAndParents"` -- active container and all ancestors +- `"CurrentPlusProjectAndShared"` -- current, project, and shared folder +- `"AllFolders"` -- everything the user can read + +### Error Handling + +```python +from labkey.exceptions import ( + RequestError, # base class for all server errors + RequestAuthorizationError, # 401 -- bad credentials + QueryNotFoundError, # 404 -- wrong schema/table name + ServerNotFoundError, # 404 -- wrong server or context_path + ServerContextError, # connection error, SSL error + UnexpectedRedirectError, # 302 -- usually http->https misconfiguration +) +from requests.exceptions import Timeout + +try: + result = api.query.select_rows("lists", "MyTable") +except QueryNotFoundError: + print("Table not found -- check schema and query names") +except RequestAuthorizationError: + print("Auth failed -- check API key or .netrc") +except ServerNotFoundError: + print("Server not found -- check domain and context_path") +except Timeout: + print("Request timed out") +except RequestError as e: + print(f"Server error: {e.message}") +``` + +### Data Modification APIs + +For completeness -- use these when analysis requires writing back results: + +- `api.query.insert_rows(schema, query, rows)` -- insert new rows +- `api.query.update_rows(schema, query, rows)` -- update rows (must include PK) +- `api.query.delete_rows(schema, query, rows)` -- delete rows (must include PK) +- `api.query.truncate_table(schema, query)` -- delete all rows +- `api.query.import_rows(schema, query, data_file=f)` -- bulk import from file +- `api.query.save_rows(commands)` -- batch multi-table operations + +All modification APIs accept `timeout=300`, `container_path=None`, `transacted=True`, and optional `audit_behavior` / `audit_user_comment`. + +## Important Considerations + +1. **max_rows defaults to -1 (ALL rows)** in `select_rows`. Always set an explicit `max_rows` for large tables to avoid pulling the entire dataset into memory. + +2. **Filter value delimiters are inconsistent**: `IN`/`NOT_IN` use semicolons (`"A;B;C"`), while `BETWEEN`/`NOT_BETWEEN` use commas (`"10,50"`). This is a historical API quirk. + +3. **columns is a comma-separated string**, not a list: `columns="Name,Age,Country"`. + +4. **Sort syntax**: comma-separated column names, prefix `-` for descending: `sort="Age,-Name"`. + +5. **WAF encoding**: `execute_sql` WAF-encodes SQL by default (since labkey v3.0.0). Requires LabKey Server v23.09+. Set `waf_encode_sql=False` for older servers. + +6. **Container path override**: Every API method accepts `container_path=` to query a different folder without creating a new connection. + +7. **CSRF tokens** are fetched automatically on the first request. This adds slight overhead to the first call. + +8. **Default timeout is 300 seconds** (5 minutes) for all query operations. + +9. **Multiple filters on the same column** are supported -- they are appended, not overwritten. + +10. **select_rows sends a GET request** to `query-getQuery.api`. `execute_sql` sends a POST to `query-executeSql.api`. + +11. **LabKey SQL is not standard SQL.** It is a SQL dialect specific to LabKey. Use `mcp__labkey__validateSQL` to check syntax before executing. Refer to LabKey documentation for dialect-specific features (e.g., lookup column traversal via `/` or `.` notation). + +## Typical Analysis Workflow + +```python +from labkey.api_wrapper import APIWrapper +from labkey.query import QueryFilter +import pandas as pd + +# 1. Connect (credentials read from ~/.netrc automatically) +api = APIWrapper("localhost:8080", "MyProject", use_ssl=False) + +# 2. Explore (or use MCP tools for interactive exploration) +schemas = api.query.get_queries("lists", include_columns=True) + +# 3. Retrieve data +result = api.query.select_rows("lists", "Participants", + columns="ParticipantId,Name,Age,Country", + filter_array=[QueryFilter("Age", "18", QueryFilter.Types.GTE)], + max_rows=1000, + sort="Age" +) + +# 4. Analyze with pandas +df = pd.DataFrame(result["rows"]) +print(df.describe()) +print(df.groupby("Country")["Age"].mean()) + +# 5. Complex queries with SQL +sql_result = api.query.execute_sql("lists", + "SELECT Country, COUNT(*) as N, AVG(Age) as AvgAge " + "FROM Participants GROUP BY Country ORDER BY N DESC" +) +summary = pd.DataFrame(sql_result["rows"]) +``` diff --git a/core/src/org/labkey/core/DataAnalysis_R.md b/core/src/org/labkey/core/DataAnalysis_R.md new file mode 100644 index 00000000000..ee6745f33b7 --- /dev/null +++ b/core/src/org/labkey/core/DataAnalysis_R.md @@ -0,0 +1,472 @@ +# LabKey Data Analysis using R + +This project is for performing data analysis against a LabKey Server instance using AI assistance. + +**Connection defaults:** The LabKey server URL and API key can be inferred from `.mcp.json` in this directory. The `url` field (minus the `/mcp` path) provides the server endpoint, and the `apikey` header value provides the authentication token. + +**Do not embed API keys in generated scripts.** Instead, ensure a `.netrc` file (Linux/Mac) or `_netrc` file (Windows) exists in the user's home directory with the server credentials. If the file does not exist, offer to create it using the API key from `.mcp.json`. The format is: + +``` +machine +login apikey +password +``` + +The `machine` value is the hostname only — no protocol (`https://`), no port, no path. For example, for `https://myserver.labkey.com:8443/labkey/mcp`, use `myserver.labkey.com`. On Linux/Mac, set permissions to 600 (`chmod 600 ~/.netrc`). On Windows, ensure the `_netrc` file is a plain file (not a "Text Document") and that a `HOME` environment variable points to the directory containing it. + +When writing R scripts, read the server URL from `.mcp.json` (via `jsonlite::fromJSON`) to pre-populate `labkey.setDefaults(baseUrl=)`, but omit `apiKey` — Rlabkey reads `.netrc` automatically. Before running any script, confirm with the data analyst: +- Is this the correct server? +- Should the URL use `http://` or `https://`? (infer from the URL scheme in `.mcp.json`) +- What container path should be used? (use MCP `listContainers` to show available options) + +Confirm all of these settings with the analyst before writing any script. + +## Online Reference Material +https://www.labkey.org/Documentation/wiki-page.view?name=rAPI + +## MCP Tools Available + +A LabKey MCP server is configured (see `.mcp.json`). Use these tools to explore the server interactively: + +| Tool | Purpose | +|---|---| +| `mcp__labkey__setContainer` | **Must be called first.** Sets the active container (project/folder) for subsequent calls. Path format: `MyProject/MyFolder` (no leading slash). | +| `mcp__labkey__whereAmIWhoAmITalkingTo` | Shows current user, server info, and active container. | +| `mcp__labkey__listContainers` | Lists all containers the user has read access to. | +| `mcp__labkey__listSchemas` | Lists all schemas in the active container. | +| `mcp__labkey__listTables` | Lists tables/queries within a schema. | +| `mcp__labkey__listColumns` | Shows column metadata (name, type, description) for a table. Also returns SQL source for saved queries. | +| `mcp__labkey__getSourceForSavedQuery` | Returns the SQL source of a saved query. | +| `mcp__labkey__validateSQL` | Validates LabKey SQL syntax without executing it. | + +### MCP Workflow + +1. Call `listContainers` to find available containers +2. Call `setContainer` with the desired container path +3. Use `listSchemas` -> `listTables` -> `listColumns` to explore the data model +4. Use `validateSQL` to check queries before running them +5. To actually retrieve data, write an R script using the `Rlabkey` package (see below) + +## Rlabkey R Package + +**Install:** `install.packages("Rlabkey")` +**Requires:** R 3.0+, LabKey Server v15.1+ +**Dependencies:** httr, jsonlite, Rcpp +**CRAN:** https://cran.r-project.org/package=Rlabkey + +### Connection Setup + +Rlabkey functions take `baseUrl` and `folderPath` as explicit arguments on every call. Use `labkey.setDefaults()` to set the server URL: + +**Preferred: `.netrc` authentication (no credentials in scripts)** + +```r +library(Rlabkey) + +# Set the server URL only — credentials are read from ~/.netrc automatically +labkey.setDefaults(baseUrl = "http://localhost:8080/") + +# Then call functions without baseUrl: +rows <- labkey.selectRows( + folderPath = "/home", + schemaName = "lists", + queryName = "MyTable" +) + +# Or pass baseUrl explicitly on every call: +rows <- labkey.selectRows( + baseUrl = "http://localhost:8080/", + folderPath = "/home", + schemaName = "lists", + queryName = "MyTable" +) +``` + +Rlabkey automatically reads credentials from `~/.netrc` (Linux/Mac) or `~/_netrc` (Windows). The `.netrc` entry should use `apikey` as the login and the API key as the password: + +``` +machine localhost +login apikey +password TheUniqueAPIKeyGeneratedForYou +``` + +The `machine` value must be the hostname only — no protocol, no port, no path. Set file permissions to 600 on Linux/Mac (`chmod 600 ~/.netrc`). Use `labkey.setCurlOptions(NETRC_FILE='/path/to/_netrc')` for a non-standard location. + +Authentication options (in order of preference): +- **.netrc file** (recommended): Create `~/.netrc` with `machine`, `login apikey`, `password ` fields (chmod 600). No credentials appear in scripts. +- **API key in code** (avoid in generated scripts): Pass to `labkey.setDefaults(apiKey=)`. Only use this for quick interactive testing, not in saved scripts. +- **Email/password**: `labkey.setDefaults(email="user@example.com", password="pass")` +- **Session key**: Pass a session key via `labkey.setDefaults(apiKey=)`. Session keys tie R access to the user's browser session context (same authorizations, impersonation state, etc.). + +To clear credentials: `labkey.setDefaults()` (called with no arguments resets all defaults). + +### Data Retrieval APIs + +#### labkey.selectRows -- Query a table/view + +```r +rows <- labkey.selectRows( + baseUrl = NULL, # str: server URL (e.g. "http://localhost:8080/") + folderPath, # str: container path (e.g. "/home" or "/MyProject/MyFolder") + schemaName, # str: e.g. "lists", "core", "study" + queryName, # str: table or query name + viewName = NULL, # str: named view to use + colSelect = NULL, # vector or comma-sep string: columns to return + maxRows = NULL, # int: max rows (NULL = ALL rows) + rowOffset = NULL, # int: rows to skip (pagination) + colSort = NULL, # str: column name prefixed with "+" or "-" + colFilter = NULL, # makeFilter() result: row filters + showHidden = FALSE, # logical: include hidden columns + colNameOpt = "caption", # str: "caption", "fieldname", or "rname" + containerFilter = NULL, # str: e.g. "CurrentAndSubfolders" + parameters = NULL, # named list: for parameterized queries + includeDisplayValues = FALSE, # logical: include lookup display values + method = "POST" # str: HTTP method ("GET" or "POST") +) +``` + +**Returns:** A data frame with `stringsAsFactors = FALSE`. Column names are determined by `colNameOpt`. + +#### labkey.executeSql -- Run LabKey SQL + +```r +rows <- labkey.executeSql( + baseUrl = NULL, # str: server URL + folderPath, # str: container path + schemaName, # str: target schema + sql, # str: LabKey SQL query + maxRows = NULL, # int: row limit + rowOffset = NULL, # int: row offset + colSort = NULL, # str: sort columns + showHidden = FALSE, # logical: include hidden columns + colNameOpt = "caption", # str: column naming option + containerFilter = NULL, # str: container scope + parameters = NULL # named list: query parameters +) +``` + +**Returns:** A data frame with `stringsAsFactors = FALSE`. + +#### labkey.getQueries -- List available tables/queries + +```r +queries <- labkey.getQueries( + baseUrl = NULL, # str: server URL + folderPath, # str: container path + schemaName # str: schema to explore +) +``` + +**Returns:** A data frame listing available queries in the schema. + +#### labkey.getQueryDetails -- Get column metadata + +```r +details <- labkey.getQueryDetails( + baseUrl = NULL, # str: server URL + folderPath, # str: container path + schemaName, # str: schema name + queryName # str: table/query name +) +``` + +**Returns:** A data frame with column metadata (name, type, caption, etc.). + +#### labkey.getSchemas -- List schemas + +```r +schemas <- labkey.getSchemas( + baseUrl = NULL, # str: server URL + folderPath # str: container path +) +``` + +**Returns:** A data frame listing available schemas. + +### Session-Based API (Alternative Style) + +Rlabkey also provides a session-based interface that wraps the direct functions: + +```r +# Create a session +s <- getSession( + baseUrl = "http://localhost:8080/", + folderPath = "/home" +) + +# Explore +lsProjects("http://localhost:8080/") # list projects (before session) +lsFolders(s) # list folders in session +lsSchemas(s) # list schemas in session + +# Get schema and retrieve data +scobj <- getSchema(s, "lists") # returns schema object with query names +df <- getRows(s, scobj$MyTable) # returns data frame (colNameOpt defaults to "fieldname") +``` + +The session-based `getRows` function defaults to `colNameOpt='fieldname'` (unlike `labkey.selectRows` which defaults to `'caption'`). + +### Response Format + +Both `labkey.selectRows` and `labkey.executeSql` return R data frames directly: + +```r +rows <- labkey.selectRows( + baseUrl = "http://localhost:8080/", + folderPath = "/home", + schemaName = "lists", + queryName = "MyTable" +) + +# The result is already a data frame: +nrow(rows) # number of rows +colnames(rows) # column names +str(rows) # structure/types +head(rows) # preview first rows + +# Access columns directly: +rows$Name +rows$Age + +# Date columns come back as strings -- convert explicitly: +rows$StartDate <- as.Date(rows$StartDate) + +# Or for datetime with timezone: +rows$Created <- as.POSIXct(rows$Created, format = "%Y/%m/%d %H:%M:%S") +``` + +### Column Name Options (`colNameOpt`) + +The `colNameOpt` parameter controls how data frame columns are named: + +| Value | Description | Example | +|---|---|---| +| `"caption"` | Field caption/label (default for `labkey.selectRows`). Best for display, harder to script with. | `"Participant ID"` | +| `"fieldname"` | Field name as used in LabKey API calls (default for `getRows`). Best for scripting. | `"ParticipantId"` | +| `"rname"` | R-safe name: lowercase, spaces become `_`, slashes become `_`. Used by LabKey R Views. | `"participantid"` | + +### Query Filters + +```r +# Build filters with makeFilter(): +filters <- makeFilter( + c("Country", "EQUAL", "Germany"), + c("Age", "GREATER_THAN_OR_EQUAL", "18"), + c("Status", "IN", "Active;Enrolled") +) +rows <- labkey.selectRows(baseUrl = "http://localhost:8080/", + folderPath = "/home", schemaName = "study", + queryName = "Demographics", colFilter = filters) +``` + +The `makeFilter()` function accepts any number of filter triplets in the form `c("column", "OPERATOR", "value")`. Multiple filters are ANDed together. + +#### Filter Operators Reference + +**Comparison:** `EQUAL`, `NOT_EQUAL`, `GREATER_THAN`, `LESS_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN_OR_EQUAL`, `NOT_EQUAL_OR_MISSING` + +**Date comparison:** `DATE_EQUAL`, `DATE_NOT_EQUAL`, `DATE_GREATER_THAN`, `DATE_LESS_THAN`, `DATE_GREATER_THAN_OR_EQUAL`, `DATE_LESS_THAN_OR_EQUAL` + +**String:** `STARTS_WITH`, `DOES_NOT_START_WITH`, `CONTAINS`, `DOES_NOT_CONTAIN`, `CONTAINS_ONE_OF`, `CONTAINS_NONE_OF` + +**Set/Range:** `IN`, `NOT_IN` (semicolon-delimited), `BETWEEN`, `NOT_BETWEEN` (comma-delimited), `MEMBER_OF` + +**Null checks (use empty string as value):** `MISSING`, `NOT_MISSING`, `MV_INDICATOR`, `NO_MV_INDICATOR` + +**Array:** `ARRAY_CONTAINS_ALL`, `ARRAY_CONTAINS_ANY`, `ARRAY_CONTAINS_NONE`, `ARRAY_CONTAINS_EXACT`, `ARRAY_CONTAINS_NOT_EXACT`, `ARRAY_ISEMPTY`, `ARRAY_ISNOTEMPTY` + +**Search:** `Q` (full-text search across table) + +**Lineage:** `EXP_CHILD_OF`, `EXP_PARENT_OF`, `EXP_LINEAGE_OF` + +**Ontology:** `ONTOLOGY_IN_SUBTREE`, `ONTOLOGY_NOT_IN_SUBTREE` + +#### Filter Examples + +```r +# Single filter (equals is common): +makeFilter(c("Country", "EQUAL", "Germany")) + +# Multiple filters (ANDed): +makeFilter( + c("TextFld", "CONTAINS", "h"), + c("BooleanFld", "EQUAL", "TRUE") +) + +# IN operator (semicolon-delimited values): +makeFilter(c("RowId", "IN", "2;3;6")) + +# MISSING operator (empty string for value): +makeFilter(c("IntFld", "MISSING", "")) +``` + +### Container Filters + +Control which containers are searched. Pass as `containerFilter=` to `labkey.selectRows`/`labkey.executeSql`: + +- `"Current"` -- only the active container (default when NULL) +- `"CurrentAndSubfolders"` -- active container and its children +- `"CurrentPlusProject"` -- active container and its parent project +- `"CurrentAndParents"` -- active container and all ancestors +- `"CurrentPlusProjectAndShared"` -- current, project, and shared folder +- `"AllFolders"` -- everything the user can read + +### Lookup Columns + +To traverse lookup (foreign key) columns, use `/` in `colSelect`: + +```r +# Include columns from a lookup target: +rows <- labkey.selectRows(baseUrl = "http://localhost:8080/", + folderPath = "/home", schemaName = "lists", + queryName = "AllTypes", + colSelect = "TextFld,IntFld,IntFld/LookupValue" +) +``` + +Use `"*"` as `colSelect` to get all columns including those not in the default view. + +### Error Handling + +Rlabkey raises R errors (via `stop()`) on failure. Use `tryCatch` for error handling: + +```r +tryCatch({ + rows <- labkey.selectRows(baseUrl = "http://localhost:8080/", + folderPath = "/home", schemaName = "lists", + queryName = "MyTable") +}, error = function(e) { + message("Error: ", e$message) +}) +``` + +Common error scenarios: +- **Wrong schema/query name**: HTTP 404 with "Query not found" message +- **Bad credentials**: HTTP 401 unauthorized +- **Wrong server URL**: Connection refused or "could not resolve host" +- **HTTP-to-HTTPS mismatch**: Unexpected redirect (302) + +For debugging, enable verbose output: +```r +labkey.setDebugMode(TRUE) +# ... run your query ... +labkey.setDebugMode(FALSE) +``` + +### WAF Encoding + +By default, Rlabkey WAF-encodes SQL in `labkey.executeSql` to pass through web application firewalls. This requires LabKey Server v23.9.0+. For older servers: + +```r +labkey.setWafEncoding(FALSE) +labkey.executeSql(baseUrl = "http://localhost:8080/", + folderPath = "/home", schemaName = "core", + sql = "SELECT * FROM Containers") +``` + +### Data Modification APIs + +For completeness -- use these when analysis requires writing back results: + +- `labkey.insertRows(baseUrl, folderPath, schemaName, queryName, toInsert)` -- insert new rows (data frame) +- `labkey.updateRows(baseUrl, folderPath, schemaName, queryName, toUpdate)` -- update rows (must include PK) +- `labkey.deleteRows(baseUrl, folderPath, schemaName, queryName, toDelete)` -- delete rows (must include PK) +- `labkey.truncateTable(baseUrl, folderPath, schemaName, queryName)` -- delete all rows +- `labkey.importRows(baseUrl, folderPath, schemaName, queryName, toImport)` -- bulk import from data frame +- `labkey.moveRows(baseUrl, folderPath, targetFolderPath, schemaName, queryName, toMove)` -- move rows to another container + +All modification APIs accept optional `provenanceParams` and `options` parameters. Common options include: +- `auditBehavior`: `"NONE"`, `"SUMMARY"`, or `"DETAILED"` +- `auditUserComment`: string attached to audit log records + +Data frames passed to modification functions must be created with `stringsAsFactors = FALSE`. Column names must match the LabKey column names. To set a value to NULL, use an empty string `""`. + +### Utility Functions + +| Function | Purpose | +|---|---| +| `labkey.whoAmI(baseUrl)` | Returns current user info (displayName, id, email, impersonated status) | +| `labkey.setDefaults(apiKey, baseUrl, email, password)` | Set default connection parameters | +| `labkey.setDebugMode(debug)` | Enable/disable debug output for requests | +| `labkey.setWafEncoding(wafEncode)` | Enable/disable WAF encoding for SQL | +| `labkey.getSchemas(baseUrl, folderPath)` | List available schemas | +| `labkey.getQueries(baseUrl, folderPath, schemaName)` | List tables/queries in a schema | +| `labkey.getQueryDetails(baseUrl, folderPath, schemaName, queryName)` | Get column metadata for a table | +| `labkey.getQueryViews(baseUrl, folderPath, schemaName, queryName)` | List named views for a table | +| `labkey.getDefaultViewDetails(baseUrl, folderPath, schemaName, queryName)` | Get default view column details | +| `labkey.getLookupDetails(baseUrl, folderPath, schemaName, queryName, lookupKey)` | Get lookup target column details | +| `labkey.getFolders(baseUrl, folderPath)` | List subfolders | + +## Important Considerations + +1. **maxRows defaults to NULL (ALL rows)** in `labkey.selectRows`. Always set an explicit `maxRows` for large tables to avoid pulling the entire dataset into memory. + +2. **Filter value delimiters are inconsistent**: `IN`/`NOT_IN` use semicolons (`"A;B;C"`), while `BETWEEN`/`NOT_BETWEEN` use commas (`"10,50"`). Null-check operators (`MISSING`, `NOT_MISSING`, etc.) require an empty string `""` as the value. + +3. **colSelect accepts both vectors and comma-separated strings**: `colSelect = c("Name", "Age")` and `colSelect = "Name,Age"` both work. When using a string, do not include spaces between column names. + +4. **Sort syntax**: prefix `+` for ascending or `-` for descending: `colSort = "+Age"` or `colSort = "-Name"`. + +5. **WAF encoding**: `labkey.executeSql` WAF-encodes SQL by default. Requires LabKey Server v23.9.0+. Call `labkey.setWafEncoding(FALSE)` for older servers. + +6. **folderPath requires a leading slash**: Use `"/home"` or `"/MyProject/MyFolder"`, not `"home"`. + +7. **colNameOpt defaults differ**: `labkey.selectRows` defaults to `"caption"`, while `getRows` (session-based) defaults to `"fieldname"`. Use `colNameOpt = "fieldname"` for consistent, scriptable column names. + +8. **Data frames for writes must use stringsAsFactors = FALSE**: When creating data frames for `labkey.insertRows`, `labkey.updateRows`, or `labkey.deleteRows`, always set `stringsAsFactors = FALSE`. + +9. **Multiple filters on the same column** are supported -- pass multiple triplets to `makeFilter()`. + +10. **baseUrl must include the context path and trailing slash**: e.g. `"http://localhost:8080/labkey/"` if the server uses a context path, or `"http://localhost:8080/"` if it does not. + +11. **LabKey SQL is not standard SQL.** It is a SQL dialect specific to LabKey. Use `mcp__labkey__validateSQL` to check syntax before executing. Refer to LabKey documentation for dialect-specific features (e.g., lookup column traversal via `/` or `.` notation). + +12. **SSL configuration**: For HTTPS servers on Windows, you may need to set the `RLABKEY_CAINFO_FILE` environment variable pointing to a CA bundle file. Use `labkey.acceptSelfSignedCerts()` for development servers with self-signed certificates. + +## Typical Analysis Workflow + +```r +library(Rlabkey) +library(jsonlite) + +# 1. Read server URL from .mcp.json (credentials come from ~/.netrc) +config <- fromJSON(".mcp.json") +server_url <- sub("/mcp$", "/", config$mcpServers$labkey$url) + +# 2. Set defaults (no apiKey — .netrc provides authentication) +labkey.setDefaults(baseUrl = server_url) + +# 3. Explore (or use MCP tools for interactive exploration) +schemas <- labkey.getSchemas(baseUrl = server_url, folderPath = "/home") +queries <- labkey.getQueries(baseUrl = server_url, folderPath = "/home", + schemaName = "lists") + +# 4. Retrieve data +rows <- labkey.selectRows(baseUrl = server_url, + folderPath = "/home", + schemaName = "lists", + queryName = "Participants", + colSelect = c("ParticipantId", "Name", "Age", "Country"), + colFilter = makeFilter(c("Age", "GREATER_THAN_OR_EQUAL", "18")), + maxRows = 1000, + colSort = "+Age", + colNameOpt = "fieldname" +) + +# 5. Analyze +summary(rows) +table(rows$Country) +tapply(rows$Age, rows$Country, mean, na.rm = TRUE) + +# 6. Complex queries with SQL +sql_result <- labkey.executeSql(baseUrl = server_url, + folderPath = "/home", + schemaName = "lists", + sql = "SELECT Country, COUNT(*) AS N, AVG(Age) AS AvgAge + FROM Participants + GROUP BY Country + ORDER BY N DESC", + colNameOpt = "fieldname" +) +print(sql_result) +``` diff --git a/core/src/org/labkey/core/login/resetPassword.jsp b/core/src/org/labkey/core/login/resetPassword.jsp index ea9e89d1c86..9a62a04ad5a 100644 --- a/core/src/org/labkey/core/login/resetPassword.jsp +++ b/core/src/org/labkey/core/login/resetPassword.jsp @@ -46,7 +46,7 @@ <% } %>

To reset your password, type in your email address and click the Reset button.

- +
<%= button("Reset").submit(true).name("reset")%> <%= button("Cancel").href(urlProvider(LoginUrls.class).getLoginURL(doneURL)) %> diff --git a/core/src/org/labkey/core/view/template/bootstrap/header.jsp b/core/src/org/labkey/core/view/template/bootstrap/header.jsp index 061a708bc82..3a0ea0ccfc6 100644 --- a/core/src/org/labkey/core/view/template/bootstrap/header.jsp +++ b/core/src/org/labkey/core/view/template/bootstrap/header.jsp @@ -154,7 +154,7 @@