Parallel Identification Reading and Relaxed file naming in input psms…

… list (#115) * parallelIdentificationReading * allow extensions and paths in the filename of the identification file * no need to save scan header info * better variable name * limit gathering scan header info * make psm lists to facilitate parallel processing * no longer paralel * fix merge conflic Co-authored-by: MICHAEL SHORTREED <[email protected]>
smith-chem-wisc · Aug 11, 2022 · 5feda8f · 5feda8f
1 parent 30001cb
commit 5feda8f
Show file tree

Hide file tree

Showing 13 changed files with 8,271 additions and 249 deletions.
diff --git a/CMD/FlashLFQExecutable.cs b/CMD/FlashLFQExecutable.cs
@@ -69,11 +69,11 @@ private static void Run(FlashLfqSettings settings)
                 .Where(f => acceptedSpectrumFileFormats.Contains(Path.GetExtension(f).ToLowerInvariant())).ToList();
 
             // check for duplicate file names (agnostic of file extension)
-            foreach (var fileName in filePaths.GroupBy(p => Path.GetFileNameWithoutExtension(p)))
+            foreach (var fileNameWithoutExtension in filePaths.GroupBy(p => Path.GetFileNameWithoutExtension(p)))
             {
-                if (fileName.Count() > 1)
+                if (fileNameWithoutExtension.Count() > 1)
                 {
-                    var types = fileName.Select(p => Path.GetFileNameWithoutExtension(p)).Distinct();
+                    var types = fileNameWithoutExtension.Select(p => Path.GetFileNameWithoutExtension(p)).Distinct();
 
                     if (!settings.Silent)
                     {
@@ -142,9 +142,9 @@ private static void Run(FlashLfqSettings settings)
 
                 foreach (var file in filePaths)
                 {
-                    string filename = Path.GetFileNameWithoutExtension(file);
+                    string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(file);
 
-                    var expDesignForThisFile = experimentalDesign[filename];
+                    var expDesignForThisFile = experimentalDesign[fileNameWithoutExtension];
                     var split = expDesignForThisFile.Split('\t');
 
                     string condition = split[1];

diff --git a/Test/SampleFiles/Parallel/20100614_Velos1_TaGe_SA_Jurkat_3.mzML b/Test/SampleFiles/Parallel/20100614_Velos1_TaGe_SA_Jurkat_3.mzML
diff --git a/Test/SampleFiles/Parallel/20100614_Velos1_TaGe_SA_Jurkat_4.mzML b/Test/SampleFiles/Parallel/20100614_Velos1_TaGe_SA_Jurkat_4.mzML
diff --git a/Test/SampleFiles/Parallel/AllPSMs.psmtsv b/Test/SampleFiles/Parallel/AllPSMs.psmtsv
diff --git a/Test/SampleFiles/Parallel/ExperimentalDesign.tsv b/Test/SampleFiles/Parallel/ExperimentalDesign.tsv
@@ -0,0 +1,3 @@
+FileName	Condition	Biorep	Fraction	Techrep
+20100614_Velos1_TaGe_SA_Jurkat_3	Default	1	1	1
+20100614_Velos1_TaGe_SA_Jurkat_4	Default	1	1	2
diff --git a/Test/SampleFiles/Parallel/withExtensionsAndLinuxPath_AllPSMs.psmtsv b/Test/SampleFiles/Parallel/withExtensionsAndLinuxPath_AllPSMs.psmtsv
diff --git a/Test/SampleFiles/Parallel/withExtensionsAndWindowsPath_AllPSMs.psmtsv b/Test/SampleFiles/Parallel/withExtensionsAndWindowsPath_AllPSMs.psmtsv
diff --git a/Test/SampleFiles/Parallel/withExtensions_AllPSMs.psmtsv b/Test/SampleFiles/Parallel/withExtensions_AllPSMs.psmtsv
diff --git a/Test/Test.cs b/Test/Test.cs
@@ -292,6 +292,162 @@ public static void TestGenericOutput()
             File.Delete(proteinsPath);
         }
 
+        [Test]
+        public static void TestParallelProcessingMetaMorpheusOutput()
+        {
+            string search = "Parallel";
+            string psmFilename = "AllPSMs.psmtsv";
+
+            var myDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, "SampleFiles", search);
+            var pathOfIdentificationFile = Path.Combine(myDirectory, psmFilename);
+            var pathOfMzml1 = Path.Combine(myDirectory, "20100614_Velos1_TaGe_SA_Jurkat_3.mzML");
+            var pathOfMzml2 = Path.Combine(myDirectory, "20100614_Velos1_TaGe_SA_Jurkat_4.mzML");
+            Assert.That(File.Exists(pathOfIdentificationFile));
+            Assert.That(File.Exists(pathOfMzml1));
+            Assert.That(File.Exists(pathOfMzml2));
+
+            string[] myargs = new string[]
+            {
+                "--rep",
+                myDirectory,
+                "--idt",
+                pathOfIdentificationFile,
+                "--ppm",
+                "5"
+            };
+
+            CMD.FlashLfqExecutable.Main(myargs);
+
+            string peaksPath = Path.Combine(myDirectory, "QuantifiedPeaks.tsv");
+            Assert.That(File.Exists(peaksPath));
+            File.Delete(peaksPath);
+
+            string peptidesPath = Path.Combine(myDirectory, "QuantifiedPeptides.tsv");
+            Assert.That(File.Exists(peptidesPath));
+            File.Delete(peptidesPath);
+
+            string proteinsPath = Path.Combine(myDirectory, "QuantifiedProteins.tsv");
+            Assert.That(File.Exists(proteinsPath));
+            File.Delete(proteinsPath);
+        }
+
+        [Test]
+        public static void TestParallelProcessingMetaMorpheusOutputWithExtensions()
+        {
+            string search = "Parallel";
+            string psmFilename = "withExtensions_AllPSMs.psmtsv";
+
+            var myDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, "SampleFiles", search);
+            var pathOfIdentificationFile = Path.Combine(myDirectory, psmFilename);
+            var pathOfMzml1 = Path.Combine(myDirectory, "20100614_Velos1_TaGe_SA_Jurkat_3.mzML");
+            var pathOfMzml2 = Path.Combine(myDirectory, "20100614_Velos1_TaGe_SA_Jurkat_4.mzML");
+            Assert.That(File.Exists(pathOfIdentificationFile));
+            Assert.That(File.Exists(pathOfMzml1));
+            Assert.That(File.Exists(pathOfMzml2));
+
+            string[] myargs = new string[]
+            {
+                "--rep",
+                myDirectory,
+                "--idt",
+                pathOfIdentificationFile,
+                "--ppm",
+                "5"
+            };
+
+            CMD.FlashLfqExecutable.Main(myargs);
+
+            string peaksPath = Path.Combine(myDirectory, "QuantifiedPeaks.tsv");
+            Assert.That(File.Exists(peaksPath));
+            File.Delete(peaksPath);
+
+            string peptidesPath = Path.Combine(myDirectory, "QuantifiedPeptides.tsv");
+            Assert.That(File.Exists(peptidesPath));
+            File.Delete(peptidesPath);
+
+            string proteinsPath = Path.Combine(myDirectory, "QuantifiedProteins.tsv");
+            Assert.That(File.Exists(proteinsPath));
+            File.Delete(proteinsPath);
+        }
+
+        [Test]
+        public static void TestParallelProcessingMetaMorpheusOutputWithExtensionsAndWindowsPath()
+        {
+            string search = "Parallel";
+            string psmFilename = "withExtensionsAndWindowsPath_AllPSMs.psmtsv";
+
+            var myDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, "SampleFiles", search);
+            var pathOfIdentificationFile = Path.Combine(myDirectory, psmFilename);
+            var pathOfMzml1 = Path.Combine(myDirectory, "20100614_Velos1_TaGe_SA_Jurkat_3.mzML");
+            var pathOfMzml2 = Path.Combine(myDirectory, "20100614_Velos1_TaGe_SA_Jurkat_4.mzML");
+            Assert.That(File.Exists(pathOfIdentificationFile));
+            Assert.That(File.Exists(pathOfMzml1));
+            Assert.That(File.Exists(pathOfMzml2));
+
+            string[] myargs = new string[]
+            {
+                "--rep",
+                myDirectory,
+                "--idt",
+                pathOfIdentificationFile,
+                "--ppm",
+                "5"
+            };
+
+            CMD.FlashLfqExecutable.Main(myargs);
+
+            string peaksPath = Path.Combine(myDirectory, "QuantifiedPeaks.tsv");
+            Assert.That(File.Exists(peaksPath));
+            File.Delete(peaksPath);
+
+            string peptidesPath = Path.Combine(myDirectory, "QuantifiedPeptides.tsv");
+            Assert.That(File.Exists(peptidesPath));
+            File.Delete(peptidesPath);
+
+            string proteinsPath = Path.Combine(myDirectory, "QuantifiedProteins.tsv");
+            Assert.That(File.Exists(proteinsPath));
+            File.Delete(proteinsPath);
+        }
+
+        [Test]
+        public static void TestParallelProcessingMetaMorpheusOutputWithExtensionsAndLinuxPath()
+        {
+            string search = "Parallel";
+            string psmFilename = "withExtensionsAndLinuxPath_AllPSMs.psmtsv";
+
+            var myDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, "SampleFiles", search);
+            var pathOfIdentificationFile = Path.Combine(myDirectory, psmFilename);
+            var pathOfMzml1 = Path.Combine(myDirectory, "20100614_Velos1_TaGe_SA_Jurkat_3.mzML");
+            var pathOfMzml2 = Path.Combine(myDirectory, "20100614_Velos1_TaGe_SA_Jurkat_4.mzML");
+            Assert.That(File.Exists(pathOfIdentificationFile));
+            Assert.That(File.Exists(pathOfMzml1));
+            Assert.That(File.Exists(pathOfMzml2));
+
+            string[] myargs = new string[]
+            {
+                "--rep",
+                myDirectory,
+                "--idt",
+                pathOfIdentificationFile,
+                "--ppm",
+                "5"
+            };
+
+            CMD.FlashLfqExecutable.Main(myargs);
+
+            string peaksPath = Path.Combine(myDirectory, "QuantifiedPeaks.tsv");
+            Assert.That(File.Exists(peaksPath));
+            File.Delete(peaksPath);
+
+            string peptidesPath = Path.Combine(myDirectory, "QuantifiedPeptides.tsv");
+            Assert.That(File.Exists(peptidesPath));
+            File.Delete(peptidesPath);
+
+            string proteinsPath = Path.Combine(myDirectory, "QuantifiedProteins.tsv");
+            Assert.That(File.Exists(proteinsPath));
+            File.Delete(proteinsPath);
+        }
+
         [Test]
         public static void TestFlashLfqExecutableWithNormalization()
         {

diff --git a/Test/Test.csproj b/Test/Test.csproj
@@ -27,6 +27,12 @@
   </ItemGroup>
 
   <ItemGroup>
+    <None Update="SampleFiles\Parallel\20100614_Velos1_TaGe_SA_Jurkat_3.mzML">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="SampleFiles\Parallel\20100614_Velos1_TaGe_SA_Jurkat_4.mzML">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Update="SampleFiles\ExperimentalDesign.tsv">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
@@ -42,6 +48,21 @@
     <None Update="SampleFiles\Morpheus\SmallCalibratible_Yeast.PSMs.tsv">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="SampleFiles\Parallel\AllPSMs.psmtsv">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="SampleFiles\Parallel\ExperimentalDesign.tsv">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="SampleFiles\Parallel\withExtensionsAndLinuxPath_AllPSMs.psmtsv">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="SampleFiles\Parallel\withExtensionsAndWindowsPath_AllPSMs.psmtsv">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="SampleFiles\Parallel\withExtensions_AllPSMs.psmtsv">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Update="SampleFiles\PeptideShaker\Default PSM Report.tabular">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>