Merge pull request #15 from aswinnnn/v0.1.6

v0.1.6

.gitignore

@@ -71,4 +71,5 @@ docs/_build/
 # Pyenv
 .python-version
 out.txt
-requirements.txt
+requirements.txt
+ptest

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pyscan"
-version = "0.1.5"
+version = "0.1.6"
 edition = "2021"
 authors = ["Aswin <[email protected]>"]
 license = "MIT"
@@ -20,10 +20,11 @@ lazy_static = "1.4.0"
 once_cell = "1.18.0"
 pep-508 = "0.3.0"
 regex = "1.7.3"
-reqwest = {version="0.11.16", features=["blocking"]}
+reqwest = {version="0.11.16"}
 serde = {version="1.0.160", features=["derive", "serde_derive"]}
 serde_json = "1.0.96"
 toml = "0.7.3"
 lenient_semver = { version = "0.4.2", features = [ "version_semver"] }
 semver = "1.0.17"
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
+futures = "0.3.28"

README.md

@@ -6,16 +6,16 @@
 
 <!-- <img src="https://media.discordapp.net/attachments/1002212458502557718/1107648562004758538/pyscan.png?width=779&height=206"> -->
 
-<img src="./assets/pyscan.png?width=679&height=206">
+<img src="./assets/pyscan-repository.png">
 
 </h4>
 
 <h5 align="center"> <i>A dependency vulnerability scanner for your python projects, straight from the terminal.</i> </h5>
 
-+ 🚀 blazingly fast scanner that can be used within large projects. (see [benchmarks](BENCHMARKS.md))
-+ 🤖 automatically finds `requirements.txt`, `pyproject.toml` or, the source code.
-+ 🧑‍💻 can be integrated into existing build processes.
-+ 💽 In its early stage, thus hasn't been battle-hardened yet. PRs and issue makers welcome.
++ can be used within large projects. (see [benchmarks](BENCHMARKS.md))
++ automatically finds dependencies either from configuration files or within source code.
++ support for poetry,hatch,filt,pdm and can be integrated into existing build processes.
++ hasn't been battle-hardened yet. PRs and issue makers welcome.
 
 ## 🕊️ Install
 
@@ -57,38 +57,38 @@ by <i>"source"</i> I mean `requirements.txt`, `pyproject.toml` or your python fi
 Note: Your docker engine/daemon should be running as pyscan utilizes the `docker create` command.  -->
 
 <br>
-
+Pyscan will find any dependencies added through poetry, hatch, filt, pdm, etc.
 Here's the order of precedence for a source/config file:
 
 + `requirements.txt`
 + `pyproject.toml`
 + your source code (`.py`)
 
-Pyscan will use `pip` to find unknown versions, otherwise [pypi.org](https://pypi.org). Still, **Make sure you version-ize your requirements** and use proper [pep-508 syntax](https://peps.python.org/pep-0508/).
+Pyscan will use your `pip` to find unknown versions, otherwise [pypi.org](https://pypi.org) for the latest version. Still, **Make sure you version-ize your requirements** and use proper [pep-508 syntax](https://peps.python.org/pep-0508/).
 
 ## Building
 
-pyscan requires a rust version of `=> v1.70`, and might be unstable on previous releases.
-There's an overview of the codebase at [architecture](./architecture/). Grateful for all the contributions so far!
+pyscan requires a rust version of `< v1.70`, and might be unstable on previous releases.
+There's an overview of the codebase at [architecture](./architecture/). Grateful for all the contributions so far.
 
 ## 🦀 How it's done
 
-pyscan uses [OSV](https://osv.dev) as its database for now. There are plans to add a few more, given its feasible.
+pyscan uses [OSV](https://osv.dev) as its database.
+
 
 pyscan doesn't make sure your code is safe from everything. Use all resources available to you like [safety](https://pypi.org/project/safety/) Dependabot, [`pip-audit`](https://pypi.org/project/pip-audit/), trivy and the likes.
 
 ## 🐰 Todo
 
 As of June 29, 2023:
 
-- [ ] Gather time to work on it (incredible task as a high schooler)
-- [ ] Multi-threading
+- [x] Gather time to work on it (incredible task as a high schooler)
+- [x] Multi-threading
 - [ ] Better display, search, filter of vulns
-- [ ] Plethora of output options (stick to >> for now)
-- [ ] ignore vulnerabilities
+- [x] Plethora of output options (stick to >> for now)
 - [x] Benchmarks
 - [x] Architecture write-up  
 
-## 🐹 Sponsor
+## 🐹 Donate
 
 While not coding, I am a broke high school student with nothing else to do. I appreciate all the help I can get.

src/display/mod.rs

@@ -1,15 +1,13 @@
-use std::{collections::HashMap, io, process::exit};
-
+use crate::parser::structs::ScannedDependency;
 use console::{style, Term};
 use once_cell::sync::Lazy;
-
-use crate::parser::structs::ScannedDependency;
+use std::{collections::HashMap, io, process::exit};
 
 static CONS: Lazy<Term> = Lazy::new(Term::stdout);
 
 pub struct Progress {
     // this progress info only contains progress info about the found vulns.
-    count: usize,
+    pub count: usize,
     current_displayed: usize,
 }
 
@@ -40,6 +38,9 @@ impl Progress {
     pub fn count_one(&mut self) {
         self.count += 1;
     }
+    pub fn end(&mut self) {
+        let _ = CONS.clear_last_lines(1);
+    }
 }
 
 pub fn display_queried(
@@ -78,12 +79,12 @@ pub fn display_queried(
             .as_str(),
         );
     } // display the safe deps
+    let _ = display_summary(&collected);
 }
 
 pub fn display_summary(collected: &Vec<ScannedDependency>) -> io::Result<()> {
+    // thing is, collected only has vulnerable dependencies, if theres a case where no vulns have been found, it will just skip this entire thing.
     if !collected.is_empty() {
-        // thing is, collected only has vulnerable dependencies, if theres a case where no vulns have been found, it will just skip this entire thing.
-
         // --- summary starts here ---
         CONS.write_line(&format!(
             "{}",
@@ -96,12 +97,19 @@ pub fn display_summary(collected: &Vec<ScannedDependency>) -> io::Result<()> {
                     "Dependency: {}",
                     style(v.name.clone()).bold().bright().red()
                 );
+
+                CONS.write_line(name.as_str())?;
+                CONS.flush()?;
 
                 // ID
                 let id = format!("ID: {}", style(vuln.id.as_str()).bold().bright().yellow());
+                CONS.write_line(id.as_str())?;
+                CONS.flush()?;
 
                 // DETAILS
                 let details = format!("Details: {}", style(vuln.details.as_str()).italic());
+                CONS.write_line(details.as_str())?;
+                CONS.flush()?;
 
                 // VERSIONS AFFECTED from ... to
                 let vers: Vec<Vec<String>> = vuln
@@ -150,10 +158,8 @@ pub fn display_summary(collected: &Vec<ScannedDependency>) -> io::Result<()> {
 
                 println!();
 
-                CONS.write_line(name.as_str())?;
-                CONS.write_line(id.as_str())?;
-                CONS.write_line(details.as_str())?;
                 CONS.write_line(version.as_str())?;
+                CONS.flush()?;
             }
         }
     } else {

src/main.rs

@@ -1,7 +1,6 @@
 use std::{path::PathBuf, process::exit};
 use clap::{Parser, Subcommand};
-use utils::PipCache;
-
+use utils::{PipCache, SysInfo};
 use std::sync::OnceLock;
 use once_cell::sync::Lazy;
 use console::style;
@@ -10,26 +9,25 @@ mod parser;
 mod scanner;
 mod docker;
 mod display;
-
 use std::env;
-
+use tokio::task;
 use crate::{utils::get_version, parser::structs::{Dependency, VersionStatus}};
 
 #[derive(Parser, Debug)]
-#[command(author="aswinnnn",version="0.1.5",about="python dependency vulnerability scanner.")]
+#[command(author="aswinnnn",version="0.1.6",about="python dependency vulnerability scanner.\n\ndo 'pyscan [subcommand] --help' for specific help.")]
 struct Cli {
-
-    /// path to source. if not provided it will use the current directory.
+    
+    /// path to source. (default: current directory)
     #[arg(long,short,default_value=None,value_name="DIRECTORY")]
     dir: Option<PathBuf>,
+
+    /// export the result to a desired format. [json]
+    #[arg(long,short, required=false, value_name="FILENAME")]
+    output: Option<String>,
 
-    /// search for a single package, do "pyscan package --help" for more
+    /// search for a single package.
     #[command(subcommand)]
     subcommand: Option<SubCommand>,
-
-    // /// scan a docker image, do "pyscan docker --help" for more
-    // #[command(subcommand)]
-    // docker: Option<SubCommand>,
 
     /// skip: skip the given databases
     /// ex. pyscan -s osv snyk
@@ -66,12 +64,12 @@ enum SubCommand {
         #[arg(long,short)]
         name: String,
 
-        /// version of the package (if not provided, the latest stable will be used)
+        /// version of the package (defaults to latest if not provided)
         #[arg(long, short, default_value=None)]
         version: Option<String>
     },
 
-    /// scan a docker image
+    /// scan inside a docker image
     Docker {
 
         /// name of the docker image
@@ -96,33 +94,23 @@ static PIPCACHE: Lazy<PipCache> = Lazy::new(|| {utils::PipCache::init()});
 // is a hashmap of package name, version from 'pip list'
 // because calling 'pip show' everytime might get expensive if theres a lot of dependencies to check. 
 
-
 #[tokio::main]
 async fn main() {
-
-    println!("pyscan v{} | by Aswin S (github.com/aswinnnn)", get_version());  
-
-    // init pip cache, if cache-off is false
-    if !&ARGS.get().unwrap().cache_off {
-        let _ = PIPCACHE.lookup("something");
-    }
-    // since its in Lazy its first accesss would init the cache, the result is ignorable.
 
     match &ARGS.get().unwrap().subcommand {
         // subcommand package
 
         Some(SubCommand::Package { name, version }) => {
             // let osv = Osv::new().expect("Cannot access the API to get the latest package version.");
-            let version = if let Some(v) = version {v.to_string()} else {utils::get_package_version_pypi(name.as_str()).expect("Error in retrieving stable version from API").to_string()};
+            let version = if let Some(v) = version {v.to_string()} else {utils::get_package_version_pypi(name.as_str()).await.expect("Error in retrieving stable version from API").to_string()};
 
             let dep = Dependency {name: name.to_string(), version: Some(version), comparator: None, version_status: VersionStatus {pypi: false, pip: false, source: false}};
 
             // start() from scanner only accepts Vec<Dependency> so
             let vdep = vec![dep];
 
-            let _res = scanner::start(vdep);
+            let _res = scanner::start(vdep).await;
             exit(0)
-
         },
         Some(SubCommand::Docker { name, path}) => {
             println!("{} {}\n{} {}",style("Docker image:").yellow().blink(),
@@ -138,8 +126,25 @@ async fn main() {
         None => ()
     }
 
+    println!("pyscan v{} | by Aswin S (github.com/aswinnnn)", get_version());  
+
+    let sys_info =  SysInfo::new().await;
+    // supposed to be a global static, cant atm because async closures are unstable.
+    // has to be ran in diff thread due to underlying blocking functions, to be fixed soon.
+
+    task::spawn(async move {
+        // init pip cache, if cache-off is false or pip has been found
+        if !&ARGS.get().unwrap().cache_off | sys_info.pip_found { 
+                let _ = PIPCACHE.lookup(" ");
+                // since its in Lazy its first accesss would init the cache, the result is ignorable.
+            }
+        // has to be run on another thread to not block user functionality
+        // it still blocks because i cant make pip_list() async or PIPCACHE would fail
+        // as async closures aren't stable yet.
+        // but it removes a 3s delay, for now.
+    });
+
 
-    // println!("{:?}", args);
 
     // --- giving control to parser starts here ---
 
@@ -151,3 +156,4 @@ async fn main() {
     else {eprintln!("the given directory is empty."); exit(1)}; // err when dir is empty
 
 }
+