From ae21882c27f9f8181512f8e69ee730876fa95f2e Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Wed, 15 Apr 2020 20:05:43 -0700 Subject: [PATCH 1/5] Mock Mascot servlet that mimics a few Mascot APIs. junit test that exercises those APIs. --- ms2/src/org/labkey/ms2/MS2Module.java | 3 +- .../ms2/pipeline/mascot/MascotClientImpl.java | 20 +++++++- .../pipeline/mascot/MockMascotServlet.java | 50 +++++++++++++++++++ .../MockMascotServletContextListener.java | 22 ++++++++ 4 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServlet.java create mode 100644 ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServletContextListener.java diff --git a/ms2/src/org/labkey/ms2/MS2Module.java b/ms2/src/org/labkey/ms2/MS2Module.java index 7af4114287..465142c683 100644 --- a/ms2/src/org/labkey/ms2/MS2Module.java +++ b/ms2/src/org/labkey/ms2/MS2Module.java @@ -24,7 +24,6 @@ import org.labkey.api.exp.ExperimentRunType; import org.labkey.api.exp.Handler; import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.property.PropertyService; import org.labkey.api.files.FileContentService; import org.labkey.api.files.TableUpdaterFileListener; import org.labkey.api.module.FolderTypeManager; @@ -61,6 +60,7 @@ import org.labkey.ms2.pipeline.comet.Comet2015ParamsBuilder; import org.labkey.ms2.pipeline.comet.CometPipelineProvider; import org.labkey.ms2.pipeline.mascot.MascotCPipelineProvider; +import org.labkey.ms2.pipeline.mascot.MascotClientImpl; import org.labkey.ms2.pipeline.rollup.FractionRollupPipelineProvider; import org.labkey.ms2.pipeline.sequest.BooleanParamsValidator; import org.labkey.ms2.pipeline.sequest.ListParamsValidator; @@ -336,6 +336,7 @@ public Set getIntegrationTests() return Set.of( Comet2014ParamsBuilder.FullParseTestCase.class, Comet2015ParamsBuilder.FullParseTestCase.class, + MascotClientImpl.TestCase.class, MS2Controller.TestCase.class, ThermoSequestParamsBuilder.TestCase.class ); diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java index 0a98d07602..c3867359a5 100644 --- a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java +++ b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java @@ -26,11 +26,15 @@ import org.apache.commons.httpclient.methods.multipart.StringPart; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.module.ModuleLoader; import org.labkey.api.ms2.SearchClient; import org.labkey.api.pipeline.ParamParser; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobService; import org.labkey.api.util.HelpTopic; +import org.labkey.api.view.ActionURL; import org.labkey.ms2.pipeline.AbstractMS2SearchProtocolFactory; import org.labkey.ms2.pipeline.AbstractMS2SearchTask; import org.labkey.ms2.pipeline.SearchFormUtil; @@ -68,7 +72,7 @@ public class MascotClientImpl implements SearchClient { - private static Logger _log = Logger.getLogger(MascotClientImpl.class); + private static final Logger _log = Logger.getLogger(MascotClientImpl.class); private Logger _instanceLogger; @@ -1509,5 +1513,19 @@ private InputStream getRequestResultStream (Properties parameters) return null; } + public static class TestCase extends Assert + { + @Test + public void testMockMascotServer() + { + MascotClientImpl client = new MascotClientImpl(ActionURL.getBaseServerURL() + "/mockmascot/cgi/", _log); + String version = client.getMascotVersion().trim(); + Assert.assertEquals("Hello - Server: LabKey MockMascotServer 1.0", version); + + String mascotSessionId = client.startSession(); + String paramFile = ModuleLoader.getInstance().getModule("MS2").getSourcePath() + "/src/org/labkey/ms2/pipeline/mascot/MascotDefaults.xml"; + Assert.assertTrue(client.submitFile(mascotSessionId, "5678", "submit.pl", paramFile, paramFile)); + } + } } diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServlet.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServlet.java new file mode 100644 index 0000000000..15286e1b66 --- /dev/null +++ b/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServlet.java @@ -0,0 +1,50 @@ +package org.labkey.ms2.pipeline.mascot; + +import javax.servlet.ServletException; +import javax.servlet.ServletOutputStream; +import javax.servlet.annotation.MultipartConfig; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * Mocks a minimal set of Mascot APIs to allow for rudimentary testing without a Mascot server. + */ +@MultipartConfig +public class MockMascotServlet extends HttpServlet +{ + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException + { + // Respond to GET mockservlet/cgi/client.pl?version + if (req.getPathInfo().equals("/cgi/client.pl") && req.getQueryString().equals("version")) + { + resp.setStatus(HttpServletResponse.SC_OK); + resp.setHeader("Server", "LabKey MockMascotServer 1.0"); + resp.getOutputStream().print("Hello"); + resp.flushBuffer(); + } + else if (req.getPathInfo().equals("/cgi/login.pl")) + { + resp.getOutputStream().print("sessionID=1234"); + resp.flushBuffer(); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException, ServletException + { + if (req.getPathInfo().equals("/cgi/submit.pl")) + { + assert req.getQueryString().equals("1+--taskID+5678+--sessionID+1234"); + assert req.getPart("FILE").getSize() == 8403; + resp.setStatus(HttpServletResponse.SC_OK); + ServletOutputStream os = resp.getOutputStream(); + os.print("Peptide #1: GWKEPA"); + os.print("Peptide #2: AQPPVTA"); + os.print("Finished uploading search details"); + resp.flushBuffer(); + } + } +} diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServletContextListener.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServletContextListener.java new file mode 100644 index 0000000000..3c48b75c23 --- /dev/null +++ b/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServletContextListener.java @@ -0,0 +1,22 @@ +package org.labkey.ms2.pipeline.mascot; + +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; +import javax.servlet.ServletRegistration; +import javax.servlet.annotation.WebListener; + +@WebListener +public class MockMascotServletContextListener implements ServletContextListener +{ + @Override + public void contextInitialized(ServletContextEvent servletContextEvent) + { + ServletRegistration.Dynamic servlet = servletContextEvent.getServletContext().addServlet("MockMascotServlet", MockMascotServlet.class); + servlet.addMapping("/mockmascot/*"); + } + + @Override + public void contextDestroyed(ServletContextEvent servletContextEvent) + { + } +} From 84b0a0f8610aa5798a4e4e12625444b7d1065c72 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Thu, 16 Apr 2020 15:32:51 -0700 Subject: [PATCH 2/5] Convert two usages of commons HttpClient 3.1 to modern HTTP libraries (one to the Java HttpClient and the other to Apache HttpComponents). Remove commons HttpClient 3.1 from the product! --- ms2/build.gradle | 1 - ms2/resources/credits/jars.txt | 4 - .../ms2/pipeline/mascot/MascotClientImpl.java | 222 +++++++++--------- .../pipeline/mascot/MockMascotServlet.java | 55 ++++- 4 files changed, 164 insertions(+), 118 deletions(-) delete mode 100644 ms2/resources/credits/jars.txt diff --git a/ms2/build.gradle b/ms2/build.gradle index 52de069718..241ef91152 100644 --- a/ms2/build.gradle +++ b/ms2/build.gradle @@ -1,7 +1,6 @@ import org.labkey.gradle.util.BuildUtils dependencies { - external 'commons-httpclient:commons-httpclient:3.1' BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "assay"), depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "experiment"), depProjectConfig: "published", depExtension: "module") diff --git a/ms2/resources/credits/jars.txt b/ms2/resources/credits/jars.txt deleted file mode 100644 index 72d7fec079..0000000000 --- a/ms2/resources/credits/jars.txt +++ /dev/null @@ -1,4 +0,0 @@ -{table} -Filename|Component|Version|Source|License|LabKey Dev|Purpose -commons-httpclient-3.1.jar|Commons HTTP Client|3.1|{link:Apache|http://jakarta.apache.org/commons/httpclient/}|{link:Apache 2.0|http://www.apache.org/licenses/LICENSE-2.0}|jeckels|HTTP client for requests of remote servers (old version) -{table} \ No newline at end of file diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java index c3867359a5..7a9ca728bc 100644 --- a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java +++ b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java @@ -17,14 +17,13 @@ package org.labkey.ms2.pipeline.mascot; import org.apache.commons.beanutils.converters.BooleanConverter; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.commons.httpclient.methods.PostMethod; -import org.apache.commons.httpclient.methods.multipart.FilePart; -import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity; -import org.apache.commons.httpclient.methods.multipart.Part; -import org.apache.commons.httpclient.methods.multipart.StringPart; import org.apache.commons.lang3.StringUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.entity.mime.content.FileBody; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Test; @@ -54,12 +53,17 @@ import java.io.PrintWriter; import java.net.HttpURLConnection; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Enumeration; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -74,7 +78,7 @@ public class MascotClientImpl implements SearchClient { private static final Logger _log = Logger.getLogger(MascotClientImpl.class); - private Logger _instanceLogger; + private final Logger _instanceLogger; private String _url; private String _userAccount; @@ -564,26 +568,27 @@ public String getMascotVersion() { mascotRequestURL = urlSB.toString(); } - GetMethod get=new GetMethod(mascotRequestURL); - HttpClient client = new HttpClient(); - String result="Sorry, unable to get Mascot version"; + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder(URI.create(mascotRequestURL)).build(); + String result = "Sorry, unable to get Mascot version"; try { - int statusCode = client.executeMethod(get); - if (statusCode == -1) { - result=result+" "+get.getResponseBodyAsString(); - } else { - result=get.getResponseBodyAsString() - +" - "+get.getResponseHeader("Server"); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() == -1) + { + result = result + " " + response.body(); } - } catch (IOException e) + else + { + result = response.body() + " - Server: " + response.headers().firstValue("Server").orElse(""); + } + } + catch (IOException | InterruptedException e) { getLogger().warn("Failed to get Mascot server information via '" + mascotRequestURL + "'", e); - } finally { - get.releaseConnection(); } - result=result.replaceAll("[\r\n]"," "); + result = result.replaceAll("[\r\n]"," "); return result; } @@ -1076,7 +1081,9 @@ protected boolean submitFile (String sessionID, String taskID, {"ltol", "mascot, ltol", "default, ltol"}, {"showallmods", "mascot, showallmods", "default, showallmods"} }; - List parts = new ArrayList<>(); + + Map parts = new LinkedHashMap<>(); + for (String [] keys : submitFields) { int j; @@ -1093,13 +1100,12 @@ protected boolean submitFile (String sessionID, String taskID, { for (String db : AbstractMS2SearchProtocolFactory.splitSequenceFiles(formFieldValue)) { - parts.add(new StringPart(formFieldKey, db)); - + parts.put(formFieldKey, db); } } else { - parts.add(new StringPart(formFieldKey, (null == formFieldValue) ? "" : formFieldValue)); + parts.put(formFieldKey, null == formFieldValue ? "" : formFieldValue); } } @@ -1122,41 +1128,30 @@ protected boolean submitFile (String sessionID, String taskID, } else parentMassError = parser.getInputParameter("search, tol"); - parts.add(new StringPart("TOL", (null==parentMassError)?"":parentMassError)); + parts.put("TOL", null==parentMassError ? "" : parentMassError); String massType = parser.getInputParameter("spectrum, fragment mass type"); if (massType == null) massType = parser.getInputParameter("search, mass"); - parts.add(new StringPart("MASS", (null==massType)?"":massType)); + parts.put("MASS", null == massType ? "" : massType); boolean isMonoisoptopicMass = "monoisotopic".equalsIgnoreCase(massType); String fragmentMassError = parser.getInputParameter(isMonoisoptopicMass ? "spectrum, fragment monoisotopic mass error" : "spectrum, fragment mass error"); if (fragmentMassError == null) { fragmentMassError = parser.getInputParameter("search, itol"); } - parts.add(new StringPart("ITOL", (null==fragmentMassError)?"":fragmentMassError)); + parts.put("ITOL", null == fragmentMassError ? "" : fragmentMassError); String fragmentMassErrorUnits = parser.getInputParameter(isMonoisoptopicMass ? "spectrum, fragment monoisotopic mass error units" : "spectrum, fragment mass error units"); if (fragmentMassErrorUnits == null) fragmentMassErrorUnits = parser.getInputParameter("search, itolu"); - parts.add(new StringPart("ITOLU", (null==fragmentMassErrorUnits)?"":fragmentMassErrorUnits)); + parts.put("ITOLU", null == fragmentMassErrorUnits ? "" : fragmentMassErrorUnits); // Decoy controlled by "mascot, decoy", submitted as "1" or nothing at all String decoyValue = parser.getInputParameter("mascot, decoy"); if (decoyValue != null && ((Boolean)new BooleanConverter().convert(Boolean.class, decoyValue)).booleanValue()) { - parts.add(new StringPart("DECOY", "1")); - } - - File queryFile = new File(analysisFile); - getLogger().info("Submitting query file, size="+queryFile.length()); - try { - parts.add(new FilePart("FILE", queryFile)); - } - catch (FileNotFoundException err) - { - getLogger().error("Failed to find Mascot query file '" + queryFile.getPath () + "'.\n"); - return false; + parts.put("DECOY", "1"); } String mascotRequestURL; @@ -1177,87 +1172,93 @@ protected boolean submitFile (String sessionID, String taskID, mascotRequestURL = urlSB.toString(); } - PostMethod post = new PostMethod(mascotRequestURL); - post.setRequestEntity(new MultipartRequestEntity(parts.toArray(new Part[0]), post.getParams()) ); - HttpClient client = new HttpClient(); - - int statusCode = -1; - int attempt = 0; - // We will retry up to 3 times. - final int maxAttempt = 3; - while (statusCode == -1 && attempt < maxAttempt) + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - try - { - // TODO: wch - we should extend StringPart and FilePart - // so that we may write to log on the amount of data transmitted - statusCode = client.executeMethod(post); - } - catch (IOException err) - { - getLogger().error("Failed to submit Mascot query '" + mascotRequestURL + "' for " + - queryFile.getPath() + " with parameters " + queryParamFile.getPath () + " on attempt#" + - (attempt + 1) + ".\n", err); - attempt = maxAttempt; - } - attempt++; - } - // Check that we didn't run out of retries. - if (statusCode == -1) { - post.releaseConnection(); - getLogger().error("Failed to submit Mascot query '" + mascotRequestURL + "' for " + - queryFile.getPath() + " with parameters " + queryParamFile.getPath() + "." + - " Tried " + maxAttempt + " times."); - return false; - } + HttpPost post = new HttpPost(mascotRequestURL); + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + parts.forEach(builder::addTextBody); - boolean uploadFinished = false; - try - { - // handle response. - // check for "Finished uploading search details..." - //for Mascot version earlier than 2.2.03 - //final String endOfUploadMarker = "Finished uploading search details..."; - //for Mascot version 2.2.03 - //final String endOfUploadMarker = "Finished uploading search details and file..."; - final String endOfUploadMarker = "Finished uploading search details"; - StringBuilder response = new StringBuilder(); - try (BufferedReader in = new BufferedReader(new InputStreamReader(post.getResponseBodyAsStream()))) + File queryFile = new File(analysisFile); + getLogger().info("Submitting query file, size="+queryFile.length()); + builder.addPart("FILE", new FileBody(queryFile)); + + post.setEntity(builder.build()); + + int attempt = 0; + // We will retry up to 3 times. + final int maxAttempt = 3; + + while (attempt < maxAttempt) { - String str; - while ((str = in.readLine()) != null) + try (CloseableHttpResponse response = httpclient.execute(post)) { - response.append(str); - response.append('\n'); - //getLogger().info("Mascot Server: "+str); - if (str.contains(endOfUploadMarker)) + if (-1 == response.getStatusLine().getStatusCode()) + continue; + + boolean uploadFinished = false; + try + { + // handle response. + // check for "Finished uploading search details..." + //for Mascot version earlier than 2.2.03 + //final String endOfUploadMarker = "Finished uploading search details..."; + //for Mascot version 2.2.03 + //final String endOfUploadMarker = "Finished uploading search details and file..."; + final String endOfUploadMarker = "Finished uploading search details"; + StringBuilder sb = new StringBuilder(); + try (BufferedReader in = new BufferedReader(new InputStreamReader(response.getEntity().getContent()))) + { + String str; + while ((str = in.readLine()) != null) + { + sb.append(str); + sb.append('\n'); + //getLogger().info("Mascot Server: "+str); + if (str.contains(endOfUploadMarker)) + { + uploadFinished = true; + getLogger().info("Mascot search task status: query upload completed"); + // Need to continue waiting for Mascot server to close the connection or it will cause the + // search to error - see issue 29773 + } + } + } + if (!uploadFinished) + { + getLogger().error("Failed to get response from Mascot query '" + mascotRequestURL + "' for " + + queryFile.getPath() + " with parameters " + queryParamFile.getPath() + " on attempt#" + + (attempt + 1) + ".\n" + "Mascot output: " + sb.toString()); + } + } + catch (IOException err) { - uploadFinished = true; - getLogger().info("Mascot search task status: query upload completed"); - // Need to continue waiting for Mascot server to close the connection or it will cause the - // search to error - see issue 29773 + getLogger().error("Failed to get response from Mascot query '" + mascotRequestURL + "' for " + + queryFile.getPath() + " with parameters " + queryParamFile.getPath() + " on attempt#" + + (attempt + 1) + ".\n", err); } + return uploadFinished; } + catch (IOException err) + { + getLogger().error("Failed to submit Mascot query '" + mascotRequestURL + "' for " + + queryFile.getPath() + " with parameters " + queryParamFile.getPath() + " on attempt#" + + (attempt + 1) + ".\n", err); + attempt = maxAttempt; + } + attempt++; } - if (!uploadFinished) - { - getLogger().error("Failed to get response from Mascot query '" + mascotRequestURL + "' for " + - queryFile.getPath() + " with parameters " + queryParamFile.getPath () + " on attempt#" + - (attempt + 1) + ".\n" + "Mascot output: " + response.toString()); - } - } - catch (IOException err) - { - getLogger().error("Failed to get response from Mascot query '" + mascotRequestURL + "' for " + - queryFile.getPath() + " with parameters " + queryParamFile.getPath () + " on attempt#" + - (attempt + 1) + ".\n",err); + + // We ran out of retries! + getLogger().error("Failed to submit Mascot query '" + mascotRequestURL + "' for " + + queryFile.getPath() + " with parameters " + queryParamFile.getPath() + "." + + " Tried " + maxAttempt + " times."); } - finally + catch (IOException e) { - post.releaseConnection(); + getLogger().error("Failed to create CloseableHttpClient", e); } - return uploadFinished; + return false; } protected boolean getResultFile (String sessionID, String taskID, String resultFile) @@ -1513,6 +1514,7 @@ private InputStream getRequestResultStream (Properties parameters) return null; } + // This test requires file access to MascotDefaults.xml, so it will run on a development machine, but not on TeamCity. public static class TestCase extends Assert { @Test diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServlet.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServlet.java index 15286e1b66..2aeddc96a8 100644 --- a/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServlet.java +++ b/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServlet.java @@ -1,5 +1,7 @@ package org.labkey.ms2.pipeline.mascot; +import org.apache.commons.io.IOUtils; + import javax.servlet.ServletException; import javax.servlet.ServletOutputStream; import javax.servlet.annotation.MultipartConfig; @@ -7,6 +9,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.nio.charset.StandardCharsets; /** * Mocks a minimal set of Mascot APIs to allow for rudimentary testing without a Mascot server. @@ -38,13 +41,59 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I if (req.getPathInfo().equals("/cgi/submit.pl")) { assert req.getQueryString().equals("1+--taskID+5678+--sessionID+1234"); + assert req.getParts().size() == 40; + testPart(req, "CHARGE", "1+, 2+ and 3+"); + testPart(req, "CLE", "Trypsin"); + testPart(req, "COM", "Comments on this Mascot search"); + testPart(req, "DB", "IPI_human_plus"); + testPart(req, "ERRORTOLERANT", "0"); + testPart(req, "FORMAT", "Mascot generic"); + testPart(req, "FORMVER", "1.01"); + testPart(req, "ICAT", ""); + testPart(req, "INSTRUMENT", "Default"); + testPart(req, "INTERMEDIATE", ""); + testPart(req, "IT_MODS", ""); + testPart(req, "MODS", ""); + testPart(req, "OVERVIEW", ""); + testPart(req, "PFA", "1"); + testPart(req, "PRECURSOR", ""); + testPart(req, "REPORT", "20"); + testPart(req, "REPTYPE", "peptide"); + testPart(req, "SEARCH", "MIS"); + testPart(req, "SEG", ""); + testPart(req, "TAXONOMY", "All entries"); + testPart(req, "TOLU", "Da"); + testPart(req, "USEREMAIL", "useremail@domain"); + testPart(req, "USERNAME", ""); + testPart(req, "IATOL", "0"); + testPart(req, "IASTOL", "0"); + testPart(req, "IA2TOL", "0"); + testPart(req, "IBTOL", "1"); + testPart(req, "IBSTOL", "0"); + testPart(req, "IB2TOL", "1"); + testPart(req, "IYTOL", "1"); + testPart(req, "IYSTOL", "0"); + testPart(req, "IY2TOL", "1"); + testPart(req, "PEAK", "auto"); + testPart(req, "LTOL", ""); + testPart(req, "SHOWALLMODS", ""); + testPart(req, "TOL", "2.0"); + testPart(req, "MASS", "Average"); + testPart(req, "ITOL", "0.8"); + testPart(req, "ITOLU", "Da"); assert req.getPart("FILE").getSize() == 8403; resp.setStatus(HttpServletResponse.SC_OK); ServletOutputStream os = resp.getOutputStream(); - os.print("Peptide #1: GWKEPA"); - os.print("Peptide #2: AQPPVTA"); - os.print("Finished uploading search details"); + os.println("Peptide #1: GWKEPA"); + os.println("Peptide #2: AQPPVTA"); + os.println("Finished uploading search details"); resp.flushBuffer(); } } + + private void testPart(HttpServletRequest req, String name, String expectedValue) throws IOException, ServletException + { + String value = IOUtils.toString(req.getPart(name).getInputStream(), StandardCharsets.US_ASCII); + assert expectedValue.equals(value); + } } From 535e8771acc24944ac14cc38f8dfad1f14c1a122 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Thu, 16 Apr 2020 16:55:35 -0700 Subject: [PATCH 3/5] Move mock Mascot servlet to devtools module Make Mascot TestCase work on TeamCity --- .../ms2/pipeline/mascot/MascotClientImpl.java | 22 ++++- .../pipeline/mascot/MockMascotServlet.java | 99 ------------------- .../MockMascotServletContextListener.java | 22 ----- 3 files changed, 19 insertions(+), 124 deletions(-) delete mode 100644 ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServlet.java delete mode 100644 ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServletContextListener.java diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java index 7a9ca728bc..e65c29f1a4 100644 --- a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java +++ b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java @@ -17,6 +17,7 @@ package org.labkey.ms2.pipeline.mascot; import org.apache.commons.beanutils.converters.BooleanConverter; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; @@ -1514,18 +1515,33 @@ private InputStream getRequestResultStream (Properties parameters) return null; } - // This test requires file access to MascotDefaults.xml, so it will run on a development machine, but not on TeamCity. + /** + * This test requires the MockMascotServlet to be running. + */ public static class TestCase extends Assert { @Test - public void testMockMascotServer() + public void testMockMascotServer() throws IOException { MascotClientImpl client = new MascotClientImpl(ActionURL.getBaseServerURL() + "/mockmascot/cgi/", _log); String version = client.getMascotVersion().trim(); Assert.assertEquals("Hello - Server: LabKey MockMascotServer 1.0", version); + // We need to POST an absolute file path to MascotDefaults.xml. First look for it in source. + String mascotDefaultsPath = MascotSearchProtocolFactory.get().getDefaultParametersResource(); + String paramFile = ModuleLoader.getInstance().getModule("MS2").getSourcePath() + "/src/" + mascotDefaultsPath; + + // Not found in source? Fine, create a temp file and use that. + if (new File(paramFile).exists()) + { + InputStream is = getClass().getClassLoader().getResourceAsStream(mascotDefaultsPath); + File file = File.createTempFile("MascotDefaults", ".xml"); + file.deleteOnExit(); + FileUtils.copyInputStreamToFile(is, file); + paramFile = file.getAbsolutePath(); + } + String mascotSessionId = client.startSession(); - String paramFile = ModuleLoader.getInstance().getModule("MS2").getSourcePath() + "/src/org/labkey/ms2/pipeline/mascot/MascotDefaults.xml"; Assert.assertTrue(client.submitFile(mascotSessionId, "5678", "submit.pl", paramFile, paramFile)); } } diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServlet.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServlet.java deleted file mode 100644 index 2aeddc96a8..0000000000 --- a/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServlet.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.labkey.ms2.pipeline.mascot; - -import org.apache.commons.io.IOUtils; - -import javax.servlet.ServletException; -import javax.servlet.ServletOutputStream; -import javax.servlet.annotation.MultipartConfig; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.nio.charset.StandardCharsets; - -/** - * Mocks a minimal set of Mascot APIs to allow for rudimentary testing without a Mascot server. - */ -@MultipartConfig -public class MockMascotServlet extends HttpServlet -{ - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException - { - // Respond to GET mockservlet/cgi/client.pl?version - if (req.getPathInfo().equals("/cgi/client.pl") && req.getQueryString().equals("version")) - { - resp.setStatus(HttpServletResponse.SC_OK); - resp.setHeader("Server", "LabKey MockMascotServer 1.0"); - resp.getOutputStream().print("Hello"); - resp.flushBuffer(); - } - else if (req.getPathInfo().equals("/cgi/login.pl")) - { - resp.getOutputStream().print("sessionID=1234"); - resp.flushBuffer(); - } - } - - @Override - protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException, ServletException - { - if (req.getPathInfo().equals("/cgi/submit.pl")) - { - assert req.getQueryString().equals("1+--taskID+5678+--sessionID+1234"); - assert req.getParts().size() == 40; - testPart(req, "CHARGE", "1+, 2+ and 3+"); - testPart(req, "CLE", "Trypsin"); - testPart(req, "COM", "Comments on this Mascot search"); - testPart(req, "DB", "IPI_human_plus"); - testPart(req, "ERRORTOLERANT", "0"); - testPart(req, "FORMAT", "Mascot generic"); - testPart(req, "FORMVER", "1.01"); - testPart(req, "ICAT", ""); - testPart(req, "INSTRUMENT", "Default"); - testPart(req, "INTERMEDIATE", ""); - testPart(req, "IT_MODS", ""); - testPart(req, "MODS", ""); - testPart(req, "OVERVIEW", ""); - testPart(req, "PFA", "1"); - testPart(req, "PRECURSOR", ""); - testPart(req, "REPORT", "20"); - testPart(req, "REPTYPE", "peptide"); - testPart(req, "SEARCH", "MIS"); - testPart(req, "SEG", ""); - testPart(req, "TAXONOMY", "All entries"); - testPart(req, "TOLU", "Da"); - testPart(req, "USEREMAIL", "useremail@domain"); - testPart(req, "USERNAME", ""); - testPart(req, "IATOL", "0"); - testPart(req, "IASTOL", "0"); - testPart(req, "IA2TOL", "0"); - testPart(req, "IBTOL", "1"); - testPart(req, "IBSTOL", "0"); - testPart(req, "IB2TOL", "1"); - testPart(req, "IYTOL", "1"); - testPart(req, "IYSTOL", "0"); - testPart(req, "IY2TOL", "1"); - testPart(req, "PEAK", "auto"); - testPart(req, "LTOL", ""); - testPart(req, "SHOWALLMODS", ""); - testPart(req, "TOL", "2.0"); - testPart(req, "MASS", "Average"); - testPart(req, "ITOL", "0.8"); - testPart(req, "ITOLU", "Da"); - assert req.getPart("FILE").getSize() == 8403; - resp.setStatus(HttpServletResponse.SC_OK); - ServletOutputStream os = resp.getOutputStream(); - os.println("Peptide #1: GWKEPA"); - os.println("Peptide #2: AQPPVTA"); - os.println("Finished uploading search details"); - resp.flushBuffer(); - } - } - - private void testPart(HttpServletRequest req, String name, String expectedValue) throws IOException, ServletException - { - String value = IOUtils.toString(req.getPart(name).getInputStream(), StandardCharsets.US_ASCII); - assert expectedValue.equals(value); - } -} diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServletContextListener.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServletContextListener.java deleted file mode 100644 index 3c48b75c23..0000000000 --- a/ms2/src/org/labkey/ms2/pipeline/mascot/MockMascotServletContextListener.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.labkey.ms2.pipeline.mascot; - -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; -import javax.servlet.ServletRegistration; -import javax.servlet.annotation.WebListener; - -@WebListener -public class MockMascotServletContextListener implements ServletContextListener -{ - @Override - public void contextInitialized(ServletContextEvent servletContextEvent) - { - ServletRegistration.Dynamic servlet = servletContextEvent.getServletContext().addServlet("MockMascotServlet", MockMascotServlet.class); - servlet.addMapping("/mockmascot/*"); - } - - @Override - public void contextDestroyed(ServletContextEvent servletContextEvent) - { - } -} From 84f8748722f68712c56d1ef43fc14fef123ff451 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Thu, 16 Apr 2020 20:01:57 -0700 Subject: [PATCH 4/5] Really make Mascot TestCase work on TeamCity --- ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java index e65c29f1a4..dd7c3c2015 100644 --- a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java +++ b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java @@ -1532,7 +1532,7 @@ public void testMockMascotServer() throws IOException String paramFile = ModuleLoader.getInstance().getModule("MS2").getSourcePath() + "/src/" + mascotDefaultsPath; // Not found in source? Fine, create a temp file and use that. - if (new File(paramFile).exists()) + if (!new File(paramFile).exists()) { InputStream is = getClass().getClassLoader().getResourceAsStream(mascotDefaultsPath); File file = File.createTempFile("MascotDefaults", ".xml"); From 6f979d4d18dc0cd6ef8cc7baa9cdc2acc8b231b6 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Fri, 17 Apr 2020 13:05:46 -0700 Subject: [PATCH 5/5] Comment out Mascot test --- ms2/src/org/labkey/ms2/MS2Module.java | 2 +- ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ms2/src/org/labkey/ms2/MS2Module.java b/ms2/src/org/labkey/ms2/MS2Module.java index 465142c683..26f3f4660b 100644 --- a/ms2/src/org/labkey/ms2/MS2Module.java +++ b/ms2/src/org/labkey/ms2/MS2Module.java @@ -336,7 +336,7 @@ public Set getIntegrationTests() return Set.of( Comet2014ParamsBuilder.FullParseTestCase.class, Comet2015ParamsBuilder.FullParseTestCase.class, - MascotClientImpl.TestCase.class, +// MascotClientImpl.TestCase.class, MS2Controller.TestCase.class, ThermoSequestParamsBuilder.TestCase.class ); diff --git a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java index dd7c3c2015..fe9ac2b642 100644 --- a/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java +++ b/ms2/src/org/labkey/ms2/pipeline/mascot/MascotClientImpl.java @@ -1516,7 +1516,8 @@ private InputStream getRequestResultStream (Properties parameters) } /** - * This test requires the MockMascotServlet to be running. + * This test requires the MockMascotServlet to be running. It was used during the migration from commons HttpClient 3.1, + * but it doesn't need to run regularly now. If needed again, it can be registered in MS2Module.java. */ public static class TestCase extends Assert {