Draft: Example nextjs site that imports repositories from GitHub to Splitgraph, then exports them to Seafowl, then renders a chart of stargazers

examples/nextjs-import-airbyte-github-export-seafowl/.env

@@ -0,0 +1,7 @@
+# This file contains public environment variables and is therefore checked into the repo
+# For secret environment variables, see `.env.local` which is _not_ checked into the repo
+# Read env-vars.d.ts for expected variable names
+# See more: https://nextjs.org/docs/app/building-your-application/configuring/environment-variables
+
+NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_REPOSITORY=github-analytics-metadata
+NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_COMPLETED_TABLE=completed_repositories

examples/nextjs-import-airbyte-github-export-seafowl/.env.test.local

@@ -0,0 +1,31 @@
+# IMPORTANT: Put your own values in `.env.local` (a git-ignored file) when running this locally
+# Configure them in Vercel settings when running in production
+# This file is mostly to show which variables exist, since it's the only one checked into the repo.
+# SEE: https://nextjs.org/docs/app/building-your-application/configuring/environment-variables
+
+# Create your own API key and secret: https://www.splitgraph.com/connect
+SPLITGRAPH_API_KEY="********************************"
+SPLITGRAPH_API_SECRET="********************************"
+
+# This should match the username associated with the API key
+NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE="*****"
+
+# Create a GitHub token that can query the repositories you want to connect
+# For example, a token with read-only access to public repos is sufficient
+# CREATE ONE HERE: https://github.com/settings/personal-access-tokens/new
+GITHUB_PAT_SECRET="github_pat_**********************_***********************************************************"
+
+# OPTIONAL: Set this environment variable to a proxy address to capture requests from API routes
+# e.g. To intercept requests to Splitgraph API sent from madatdata libraries in API routes
+# You can also set this by running: yarn dev-mitm (see package.json)
+# MITMPROXY_ADDRESS="http://localhost:7979"
+
+# OPTIONAL: Set Seafowl environment variables to use for creating fallback tables when exports fail
+# NOTE 1: At the moment the instance URL must be https://demo.seafowl.cloud because that's where
+# the Splitgraph export API exports tables to when no instance URL is specified, and we are
+# currently not specifying the instance URL when starting exports, and only use it when creating fallback tables.
+# NOTE 2: The dbname (SEAFOWL_INSTANCE_DATABASE) MUST match NEXT_PUBLIC_SPLITGRAPH_GITHUB_ANALYTICS_META_NAMESPACE
+#
+# SEAFOWL_INSTANCE_URL="https://demo.seafowl.cloud"
+# SEAFOWL_INSTANCE_SECRET="********************************"
+# SEAFOWL_INSTANCE_DATABASE="**********"

examples/nextjs-import-airbyte-github-export-seafowl/.gitignore

@@ -0,0 +1,4 @@
+.next
+.env.local
+.env.*.local
+!.env.test.local

examples/nextjs-import-airbyte-github-export-seafowl/README.md

@@ -0,0 +1,26 @@
+# End-to-End Example: Use `airbyte-github` to import GitHub repository into Splitgraph, then export it to Seafowl, via Next.js API routes
+
+This is a full end-to-end example demonstrating importing data to Splitgraph
+(using the `airbyte-github` plugin), exporting it to Seafowl (using the
+`export-to-seafowl` plugin), and then querying it (with `DbSeafowl` and React
+hooks from `@madatdata/react`). The importers and exporting of data is triggered
+by backend API routes (e.g. the Vecel runtime), which execute in an environment
+with secrets (an `API_SECRET` for Splitgraph, and a GitHub access token for
+`airbyte-github`). The client side queries Seafowl directly by sending raw SQL
+queries in HTP requests, which is what Seafowl is ultimately designed for.
+
+## Try Now
+
+### Preview Immediately
+
+_No signup required, just click the button!_
+
+[![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/splitgraph/madatdata/tree/main/examples/nextjs-import-airbyte-github-export-seafowl?file=pages/index.tsx)
+
+[![Open with CodeSandbox](https://assets.codesandbox.io/github/button-edit-lime.svg)](https://codesandbox.io/p/sandbox/github/splitgraph/madatdata/main/examples/nextjs-import-airbyte-github-export-seafowl?file=pages/index.tsx&hardReloadOnChange=true&startScript=dev&node=16&port=3000)
+
+### Or, deploy to Vercel (signup required)
+
+_Signup, fork the repo, and import it_
+
+[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/git/external?repository-url=https://github.com/splitgraph/madatdata/tree/main/examples/nextjs-import-airbyte-github-export-seafowl&project-name=madatdata-basic-hooks&repository-name=madatdata-nextjs-basic-hooks)

examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.module.css

@@ -0,0 +1,38 @@
+.container {
+  display: flex;
+  flex-direction: column;
+  height: 100vh;
+  background-color: var(--background);
+}
+
+.header {
+  width: 100%;
+  position: sticky;
+  top: 0;
+  z-index: 100;
+  background-color: var(--header);
+  color: var(--text);
+}
+
+.main {
+  display: flex;
+  flex-grow: 1;
+  overflow: hidden;
+}
+
+.sidebar {
+  width: 20%; /* adjust as per your needs */
+  overflow-y: auto;
+  /* add additional styles for your sidebar */
+  color: var(--text);
+}
+
+.content {
+  width: 80%; /* adjust as per your needs */
+  overflow-y: auto;
+  position: relative;
+  /* add additional styles for your content area */
+  color: var(--text);
+  background-color: var(--background);
+  padding: 24px;
+}

examples/nextjs-import-airbyte-github-export-seafowl/components/BaseLayout.tsx

@@ -0,0 +1,21 @@
+import styles from "./BaseLayout.module.css";
+import { Header } from "./Header";
+
+export const BaseLayout = ({
+  children,
+  sidebar,
+}: React.PropsWithChildren<{
+  sidebar: React.ReactNode;
+}>) => {
+  return (
+    <div className={styles.container}>
+      <div className={styles.header}>
+        <Header />
+      </div>
+      <div className={styles.main}>
+        <div className={styles.sidebar}>{sidebar}</div>
+        <div className={styles.content}>{children}</div>
+      </div>
+    </div>
+  );
+};

examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedPreviews.module.css

@@ -0,0 +1,13 @@
+.embeddedPreviewHeading {
+  margin-bottom: 0;
+}
+
+.embeddedPreviewDescription {
+  margin-bottom: 1rem;
+}
+
+.note {
+  font-size: small;
+  /* color: red !important; */
+  display: block;
+}

examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedPreviews.tsx

@@ -0,0 +1,153 @@
+import styles from "./EmbeddedPreviews.module.css";
+import type {
+  ExportTable,
+  ExportQueryInput,
+  ExportTableInput,
+} from "../../types";
+
+import { EmbedPreviewTableOrQuery } from "../EmbeddedQuery/EmbeddedQuery";
+import { ComponentProps } from "react";
+
+export const EmbeddedTablePreviewHeadingAndDescription = ({
+  exportComplete,
+}: {
+  exportComplete: boolean;
+}) => {
+  return (
+    <>
+      {!exportComplete ? (
+        <>
+          <h2 className={styles.embeddedPreviewHeading}>Tables to Export</h2>
+          <p className={styles.embeddedPreviewDescription}>
+            These are the tables that we'll export from Splitgraph to Seafowl.
+            You can query them in Splitgraph now, and then when the export is
+            complete, you'll be able to query them in Seafowl too.
+          </p>
+        </>
+      ) : (
+        <>
+          <h2 className={styles.embeddedPreviewHeading}>Exported Tables</h2>
+          <p className={styles.embeddedPreviewDescription}>
+            We successfully exported the tables to Seafowl, so now you can query
+            them in Seafowl too.
+          </p>
+        </>
+      )}
+    </>
+  );
+};
+
+export const EmbeddedTablePreviews = ({
+  tablesToExport,
+  splitgraphRepository,
+  splitgraphNamespace,
+  useLoadingOrCompleted,
+}: {
+  tablesToExport: ExportTableInput[];
+  splitgraphRepository: string;
+  splitgraphNamespace: string;
+  useLoadingOrCompleted: ComponentProps<
+    typeof EmbedPreviewTableOrQuery
+  >["useLoadingOrCompleted"];
+}) => {
+  return (
+    <>
+      {tablesToExport.map((exportTable) => (
+        <EmbedPreviewTableOrQuery
+          key={`export-table-preview-${exportTable.table}`}
+          exportInput={exportTable}
+          importedRepository={{ splitgraphNamespace, splitgraphRepository }}
+          makeQuery={({ splitgraphNamespace, splitgraphRepository, table }) =>
+            `SELECT * FROM "${splitgraphNamespace}/${splitgraphRepository}"."${table}";`
+          }
+          useLoadingOrCompleted={useLoadingOrCompleted}
+          makeMatchInputToExported={(exportTableInput) => (exportTable) => {
+            return (
+              exportTable.destinationSchema === exportTableInput.repository &&
+              exportTable.destinationTable === exportTableInput.table
+            );
+          }}
+        />
+      ))}
+    </>
+  );
+};
+
+export const EmbeddedQueryPreviewHeadingAndDescription = ({
+  exportComplete,
+}: {
+  exportComplete: boolean;
+}) => {
+  return (
+    <>
+      {" "}
+      {!exportComplete ? (
+        <>
+          <h2 className={styles.embeddedPreviewHeading}>Queries to Export</h2>
+          <p className={styles.embeddedPreviewDescription}>
+            We've prepared a few queries to export from Splitgraph to Seafowl,
+            so that we can use them to render the charts that we want.
+            Splitgraph will execute the query and insert its result into
+            Seafowl. You can query them in Splitgraph now, and then when the
+            export is complete, you'll be able to query them in Seafowl too.
+          </p>
+        </>
+      ) : (
+        <>
+          <h2 className={styles.embeddedPreviewHeading}>Exported Queries</h2>
+          <p className={styles.embeddedPreviewDescription}>
+            We successfully exported these queries from Splitgraph to Seafowl,
+            so now you can query them in Seafowl too.{" "}
+            <em className={styles.note}>
+              Note: If some queries failed to export, it's probably because they
+              had empty result sets (e.g. the table of issue reactions)
+            </em>
+          </p>
+        </>
+      )}
+    </>
+  );
+};
+
+export const EmbeddedQueryPreviews = ({
+  queriesToExport,
+  splitgraphRepository,
+  splitgraphNamespace,
+  useLoadingOrCompleted,
+}: {
+  queriesToExport: ExportQueryInput[];
+  splitgraphRepository: string;
+  splitgraphNamespace: string;
+  useLoadingOrCompleted: ComponentProps<
+    typeof EmbedPreviewTableOrQuery
+  >["useLoadingOrCompleted"];
+}) => {
+  return (
+    <>
+      {queriesToExport.map((exportQuery) => (
+        <EmbedPreviewTableOrQuery
+          key={`export-query-preview-${exportQuery.destinationTable}-${exportQuery.destinationSchema}`}
+          exportInput={exportQuery}
+          importedRepository={{ splitgraphNamespace, splitgraphRepository }}
+          // This is the query we run on Splitgraph that we exported to Seafowl
+          makeQuery={({ sourceQuery }) => sourceQuery}
+          // But once it's exported, we can just select from its table in Seafowl (and
+          // besides, the sourceQuery might not be compatible with Seafowl anyway)
+          makeSeafowlQuery={({ destinationSchema, destinationTable }) =>
+            `SELECT * FROM "${destinationSchema}"."${destinationTable}";`
+          }
+          useLoadingOrCompleted={useLoadingOrCompleted}
+          makeMatchInputToExported={(exportQueryInput) =>
+            (exportTable: ExportTable) => {
+              return (
+                exportTable.destinationSchema ===
+                  exportQueryInput.destinationSchema &&
+                exportTable.destinationTable ===
+                  exportQueryInput.destinationTable
+              );
+            }}
+        />
+      ))}
+    </>
+  );
+};

examples/nextjs-import-airbyte-github-export-seafowl/components/EmbeddedQuery/EmbeddedQuery.module.css

@@ -0,0 +1,45 @@
+.embeddedQuery {
+  border-left: 4px solid var(--muted);
+  padding-left: 10px;
+  margin-bottom: 2rem;
+}
+
+.topBar {
+  display: flex;
+  flex-direction: row;
+  justify-content: space-between;
+  align-items: center;
+  margin-bottom: 8px;
+}
+
+.consoleFlavorButtonsAndLoadingBar {
+  display: inline-flex;
+  flex-direction: row;
+}
+
+.embeddedQuery iframe {
+  width: 100%;
+}
+
+.heading {
+  font-size: 1.5rem;
+  font-weight: bold;
+  display: inline-flex;
+  align-items: center;
+  padding-top: 10px;
+  padding-bottom: 10px;
+}
+
+.embedControls {
+  background: inherit;
+}
+
+.openInConsoleLink {
+  display: inline-flex;
+  align-items: center;
+  font-size: small;
+}
+
+.openInConsoleLink svg {
+  margin-right: 4px;
+}