registry: fetch & serve GitHub READMEs (#481)

registry/migrations/20231011184807_add-readme-table.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS readmes (
+    id BIGSERIAL PRIMARY KEY,
+    extension_id INT4 UNIQUE NOT NULL,
+    readme_html TEXT NOT NULL
+);

registry/sqlx-data.json

@@ -612,6 +612,19 @@
     },
     "query": "SELECT id FROM extensions WHERE name = $1"
   },
+  "65a10cf0ab7fbc58df8e87fb31c2187d7696490c598f808502b35dcd3e235215": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Left": [
+          "Int4",
+          "Text"
+        ]
+      }
+    },
+    "query": "INSERT INTO readmes (extension_id, readme_html)\n            VALUES ($1, $2)\n            ON CONFLICT (extension_id)\n            DO UPDATE SET readme_html = excluded.readme_html"
+  },
   "69ef7c7c79e69f31731a41417a0047562f0806a7c73eb4bb98c9ed554fff3b7c": {
     "describe": {
       "columns": [],
@@ -942,6 +955,32 @@
     },
     "query": "DELETE FROM versions\n             WHERE extension_id = $1"
   },
+  "a58bdc12504edda01f8c370c35cb064147fd14e6767f1ed3939f5bea60bf79fa": {
+    "describe": {
+      "columns": [
+        {
+          "name": "id",
+          "ordinal": 0,
+          "type_info": "Int8"
+        },
+        {
+          "name": "repository",
+          "ordinal": 1,
+          "type_info": "Varchar"
+        }
+      ],
+      "nullable": [
+        false,
+        true
+      ],
+      "parameters": {
+        "Left": [
+          "Text"
+        ]
+      }
+    },
+    "query": "SELECT id, repository FROM extensions WHERE extensions.name = $1"
+  },
   "ac75c8170d5f5bc2fa5c48cdec0ec8899a2cfd1fd679e3656fe95c19650e5bd3": {
     "describe": {
       "columns": [
@@ -1053,6 +1092,26 @@
     },
     "query": "SELECT * FROM versions WHERE extension_id = $1 AND num = $2"
   },
+  "b5d80515c50725844a2322e6d15afe4b2a66b9809d111550a0170b355419c451": {
+    "describe": {
+      "columns": [
+        {
+          "name": "readme_html",
+          "ordinal": 0,
+          "type_info": "Text"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Left": [
+          "Text"
+        ]
+      }
+    },
+    "query": "SELECT r.readme_html \n            FROM readmes AS r \n            JOIN extensions AS e ON r.extension_id = e.id \n            WHERE e.name = $1"
+  },
   "bd184a09a0f45acc606a45ef90e0e41c500128881c1284374823f53c978b527d": {
     "describe": {
       "columns": [],

registry/src/config.rs

@@ -10,6 +10,7 @@ pub struct Config {
     pub aws_secret_key: String,
     pub auth_token: HeaderValue,
     pub clerk_secret_key: String,
+    pub github_token: String,
 }
 
 impl Default for Config {
@@ -25,6 +26,7 @@ impl Default for Config {
             aws_secret_key: from_env_default("AWS_SECRET_KEY", ""),
             auth_token: from_env_default("AUTH_TOKEN", "").parse().unwrap(),
             clerk_secret_key: env::var("CLERK_SECRET_KEY").expect("CLERK_SECRET_KEY not set"),
+            github_token: env::var("GITHUB_TOKEN").expect("GITHUB_TOKEN not set"),
         }
     }
 }

registry/src/errors.rs

@@ -10,6 +10,8 @@ use std::string::FromUtf8Error;
 use thiserror::Error;
 use url::ParseError;
 
+pub type Result<T = ()> = std::result::Result<T, ExtensionRegistryError>;
+
 // Use default implementation for `error_response()` method
 impl actix_web::error::ResponseError for ExtensionRegistryError {
     fn status_code(&self) -> reqwest::StatusCode {

registry/src/extensions.rs

@@ -84,10 +84,10 @@ pub async fn latest_license(
 
 pub async fn get_extension_id(
     extension_name: &str,
-    conn: Data<Pool<Postgres>>,
+    conn: &Pool<Postgres>,
 ) -> Result<i64, ExtensionRegistryError> {
     let id = sqlx::query!("SELECT id FROM extensions WHERE name = $1", extension_name)
-        .fetch_one(conn.as_ref())
+        .fetch_one(conn)
         .await?;
 
     Ok(id.id)

registry/src/lib.rs

@@ -6,6 +6,7 @@ pub mod config;
 pub mod download;
 pub mod errors;
 pub mod extensions;
+pub mod readme;
 pub mod repository;
 pub mod routes;
 pub mod token;

registry/src/readme.rs

@@ -0,0 +1,142 @@
+use reqwest::Client;
+
+use crate::{
+    errors::{ExtensionRegistryError, Result},
+    repository::Registry,
+};
+
+pub struct GithubApiClient {
+    token: String,
+    client: Client,
+}
+
+impl GithubApiClient {
+    pub fn new(token: String) -> Self {
+        Self {
+            token,
+            client: Client::new(),
+        }
+    }
+
+    pub async fn fetch_readme(&self, project_url: &str) -> Result<String> {
+        // TODO: deal with error
+        let project = GitHubProject::parse_url(project_url).unwrap();
+
+        let readme_url = project.build_readme_url();
+
+        self.client
+            .get(readme_url)
+            .header("Accept", "application/vnd.github.html")
+            .header("X-GitHub-Api-Version", "2022-11-28")
+            .header("User-Agent", "request")
+            .bearer_auth(&self.token)
+            .send()
+            .await?
+            .text()
+            .await
+            .map_err(Into::into)
+    }
+}
+
+#[derive(Debug, PartialEq)]
+struct GitHubProject<'a> {
+    owner: &'a str,
+    name: &'a str,
+    subdir: Option<&'a str>,
+}
+
+impl<'a> GitHubProject<'a> {
+    pub fn parse_url(url: &'a str) -> Option<Self> {
+        let remaining = url.strip_prefix("https://github.com/")?;
+
+        let mut parts = remaining.split('/');
+        let owner = parts.next()?;
+        let name = parts.next()?;
+        let subdir = if let Some("tree") = parts.next() {
+            parts.last()
+        } else {
+            None
+        };
+
+        Some(Self {
+            owner,
+            name,
+            subdir,
+        })
+    }
+
+    fn build_readme_url(&self) -> String {
+        let Self {
+            owner,
+            name,
+            subdir,
+        } = *self;
+
+        match subdir {
+            Some(subdir) if owner != "postgres" => {
+                format!("https://api.github.com/repos/{owner}/{name}/readme/{subdir}")
+            }
+            _ => format!("https://api.github.com/repos/{owner}/{name}/readme"),
+        }
+    }
+}
+
+pub async fn fetch_and_save_readme(
+    client: &GithubApiClient,
+    registry: &Registry,
+    extension_name: &str,
+) -> Result {
+    let (extension_id, extension_url) = registry.get_repository_url(extension_name).await?;
+
+    let url = extension_url.ok_or(ExtensionRegistryError::ResourceNotFound)?;
+
+    let readme = client.fetch_readme(&url).await?;
+    registry.upsert_readme(extension_id, &readme).await?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::readme::GitHubProject;
+
+    #[test]
+    fn parses_github_urls() {
+        let pgmq = "https://github.com/tembo-io/pgmq";
+        let auth_delay = "https://github.com/postgres/postgres/tree/master/contrib/auth_delay";
+
+        assert_eq!(
+            GitHubProject::parse_url(pgmq).unwrap(),
+            GitHubProject {
+                owner: "tembo-io",
+                name: "pgmq",
+                subdir: None
+            }
+        );
+        assert_eq!(
+            GitHubProject::parse_url(auth_delay).unwrap(),
+            GitHubProject {
+                owner: "postgres",
+                name: "postgres",
+                subdir: Some("auth_delay")
+            }
+        );
+    }
+
+    #[test]
+    fn builds_readme_urls() {
+        let pgmq = "https://github.com/tembo-io/pgmq";
+        let auth_delay = "https://github.com/postgres/postgres/tree/master/contrib/auth_delay";
+
+        assert_eq!(
+            GitHubProject::parse_url(pgmq).unwrap().build_readme_url(),
+            "https://api.github.com/repos/tembo-io/pgmq/readme"
+        );
+        assert_eq!(
+            GitHubProject::parse_url(auth_delay)
+                .unwrap()
+                .build_readme_url(),
+            "https://api.github.com/repos/postgres/postgres/readme"
+        );
+    }
+}

registry/src/repository.rs

@@ -4,6 +4,9 @@ use sqlx::{postgres::PgPoolOptions, Pool, Postgres};
 
 use crate::{conn_options, errors};
 
+/// Queries related to fetching information on extensions
+mod extension;
+
 /// Queries related to removing an extension from the DB
 mod remove_extension;
 

registry/src/repository/extension.rs

@@ -0,0 +1,45 @@
+use super::Registry;
+use crate::errors::Result;
+
+impl Registry {
+    pub async fn get_extension_readme(&self, extension_name: &str) -> Result<String> {
+        let record = sqlx::query!(
+            "SELECT r.readme_html 
+            FROM readmes AS r 
+            JOIN extensions AS e ON r.extension_id = e.id 
+            WHERE e.name = $1",
+            extension_name
+        )
+        .fetch_one(&self.pool)
+        .await?;
+
+        Ok(record.readme_html)
+    }
+
+    /// Fetch the repository of the extension with the given name
+    pub async fn get_repository_url(&self, extension_name: &str) -> Result<(i64, Option<String>)> {
+        let record = sqlx::query!(
+            "SELECT id, repository FROM extensions WHERE extensions.name = $1",
+            extension_name
+        )
+        .fetch_one(&self.pool)
+        .await?;
+
+        Ok((record.id, record.repository))
+    }
+
+    pub async fn upsert_readme(&self, extension_id: i64, readme_html: &str) -> Result {
+        sqlx::query!(
+            "INSERT INTO readmes (extension_id, readme_html)
+            VALUES ($1, $2)
+            ON CONFLICT (extension_id)
+            DO UPDATE SET readme_html = excluded.readme_html",
+            extension_id as i32,
+            readme_html
+        )
+        .execute(&self.pool)
+        .await?;
+
+        Ok(())
+    }
+}

registry/src/routes.rs

@@ -1,5 +1,6 @@
 pub mod categories;
 pub mod download;
 pub mod extensions;
+pub mod readmes;
 pub mod root;
 pub mod token;

registry/src/routes/download.rs

@@ -19,7 +19,7 @@ pub async fn download(
     // TODO(ianstanton) Increment download count for extension
     // Use latest version if 'latest' provided as version
     if version == "latest" {
-        let extension_id = get_extension_id(&name, conn.clone()).await?;
+        let extension_id = get_extension_id(&name, conn.as_ref()).await?;
         version = latest_version(extension_id as _, conn).await?;
     }
     let url = extension_location(&cfg.bucket_name, &name, &version);

registry/src/routes/readmes.rs

@@ -0,0 +1,26 @@
+use crate::{errors::Result, readme::GithubApiClient, repository::Registry};
+use actix_web::{get, post, web, HttpResponse};
+
+#[post("/extensions/details/{extension_name}/readme")]
+pub async fn fetch_and_save_readme(
+    path: web::Path<String>,
+    registry: web::Data<Registry>,
+    client: web::Data<GithubApiClient>,
+) -> Result<HttpResponse> {
+    let extension_name = path.into_inner();
+    crate::readme::fetch_and_save_readme(client.as_ref(), registry.as_ref(), &extension_name)
+        .await?;
+
+    Ok(HttpResponse::Ok().finish())
+}
+
+#[get("/extensions/details/{extension_name}/readme")]
+pub async fn get_readme(
+    path: web::Path<String>,
+    registry: web::Data<Registry>,
+) -> Result<HttpResponse> {
+    let extension_name = path.into_inner();
+    let readme = registry.get_extension_readme(&extension_name).await?;
+
+    Ok(HttpResponse::Ok().body(readme))
+}

registry/src/server.rs

@@ -1,6 +1,7 @@
 use actix_cors::Cors;
 use actix_web::{web, App, HttpServer};
 use clerk_rs::{validators::actix::ClerkMiddleware, ClerkConfiguration};
+use trunk_registry::readme::GithubApiClient;
 use trunk_registry::repository::Registry;
 use trunk_registry::routes::token::new_token;
 use trunk_registry::{config, connect, routes};
@@ -58,6 +59,9 @@ pub async fn server() -> std::io::Result<()> {
             .app_data(web::Data::new(registry.clone()))
             .app_data(web::Data::new(cfg.clone()))
             .app_data(web::Data::new(aws_config.clone()))
+            .app_data(web::Data::new(GithubApiClient::new(
+                cfg.github_token.clone(),
+            )))
             .configure(routes_config)
     })
     .bind(("0.0.0.0", 8080))?

registry/tests/integration_test.rs

@@ -32,6 +32,7 @@ mod tests {
 
         let dummy_jwt = "Bearer eyJhbGciOiJIUzI1NiIsImtpZCI6Imluc18yTzgzQnVQM2ZvS3dHc1o3Tks5b1pVT0lrNkQiLCJ0eXAiOiJKV1QifQ.eyJhenAiOiJodHRwOi8vbG9jYWxob3N0OjMwMDAiLCJleHAiOjE2ODM1OTU0ODMsImlhdCI6MTY4MzU5NTQyMywiaXNzIjoiaHR0cHM6Ly9lbGVjdHJpYy1jcmFwcGllLTkyLmNsZXJrLmFjY291bnRzLmRldiIsImp0aSI6Ijg3ZTFjOTc5MTBmYzA5N2E1MDlkIiwibmJmIjoxNjgzNTk1NDEzLCJzaWQiOiJzZXNzXzJQWEZHRU9pSWJvM2U5cUpqYk01c3BkdW1teSIsInN1YiI6InVzZXJfMlBIbVgzWVBqbmpOV1VsMTZMR1FUbGR1bW15IiwidXNlck5hbWUiOiJkdW1teSJ9.a70cMX7g_asjO4O5oG3ym16KTyuGRsy21fHScriZms0";
 
+        std::env::set_var("GITHUB_TOKEN", "dummy");
         let cfg = trunk_registry::config::Config::default();
         let conn = connect(&cfg.database_url)
             .await