Skip to content
This repository has been archived by the owner on Oct 19, 2024. It is now read-only.

Commit

Permalink
Optimize user-generated images for reduced bandwidth
Browse files Browse the repository at this point in the history
  • Loading branch information
@Geometrically
Geometrically committed Sep 7, 2024
1 parent cb0f03c commit 82de2ac
Show file tree
Hide file tree
Showing 20 changed files with 956 additions and 771 deletions.
46 changes: 45 additions & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ sentry-actix = "0.32.1"

image = "0.24.6"
color-thief = "0.2.2"
webp = "0.3.0"

woothee = "0.13.0"

Expand Down
22 changes: 22 additions & 0 deletions migrations/20240907192840_raw-images.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
ALTER TABLE mods ADD COLUMN raw_icon_url TEXT NULL;
UPDATE mods SET raw_icon_url = icon_url;

ALTER TABLE users ADD COLUMN raw_avatar_url TEXT NULL;
UPDATE users SET raw_avatar_url = avatar_url;

ALTER TABLE oauth_clients ADD COLUMN raw_icon_url TEXT NULL;
UPDATE oauth_clients SET raw_icon_url = icon_url;

ALTER TABLE organizations ADD COLUMN raw_icon_url TEXT NULL;
UPDATE organizations SET raw_icon_url = icon_url;

ALTER TABLE collections ADD COLUMN raw_icon_url TEXT NULL;
UPDATE collections SET raw_icon_url = icon_url;

ALTER TABLE mods_gallery ADD COLUMN raw_image_url TEXT NULL;
UPDATE mods_gallery SET raw_image_url = image_url;
ALTER TABLE mods_gallery ALTER COLUMN raw_image_url SET NOT NULL;

ALTER TABLE uploaded_images ADD COLUMN raw_url TEXT NULL;
UPDATE uploaded_images SET raw_url = url;
ALTER TABLE uploaded_images ALTER COLUMN raw_url SET NOT NULL;
10 changes: 7 additions & 3 deletions src/database/models/collection_item.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ impl CollectionBuilder {
created: Utc::now(),
updated: Utc::now(),
icon_url: None,
raw_icon_url: None,
color: None,
status: self.status,
projects: self.projects,
Expand All @@ -51,6 +52,7 @@ pub struct Collection {
pub created: DateTime<Utc>,
pub updated: DateTime<Utc>,
pub icon_url: Option<String>,
pub raw_icon_url: Option<String>,
pub color: Option<u32>,
pub status: CollectionStatus,
pub projects: Vec<ProjectId>,
Expand All @@ -65,11 +67,11 @@ impl Collection {
"
INSERT INTO collections (
id, user_id, name, description,
created, icon_url, status
created, icon_url, raw_icon_url, status
)
VALUES (
$1, $2, $3, $4,
$5, $6, $7
$5, $6, $7, $8
)
",
self.id as CollectionId,
Expand All @@ -78,6 +80,7 @@ impl Collection {
self.description.as_ref(),
self.created,
self.icon_url.as_ref(),
self.raw_icon_url.as_ref(),
self.status.to_string(),
)
.execute(&mut **transaction)
Expand Down Expand Up @@ -165,7 +168,7 @@ impl Collection {
let collections = sqlx::query!(
"
SELECT c.id id, c.name name, c.description description,
c.icon_url icon_url, c.color color, c.created created, c.user_id user_id,
c.icon_url icon_url, c.raw_icon_url raw_icon_url, c.color color, c.created created, c.user_id user_id,
c.updated updated, c.status status,
ARRAY_AGG(DISTINCT cm.mod_id) filter (where cm.mod_id is not null) mods
FROM collections c
Expand All @@ -183,6 +186,7 @@ impl Collection {
name: m.name.clone(),
description: m.description.clone(),
icon_url: m.icon_url.clone(),
raw_icon_url: m.raw_icon_url.clone(),
color: m.color.map(|x| x as u32),
created: m.created,
updated: m.updated,
Expand Down
12 changes: 8 additions & 4 deletions src/database/models/image_item.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ const IMAGES_NAMESPACE: &str = "images";
pub struct Image {
pub id: ImageId,
pub url: String,
pub raw_url: String,
pub size: u64,
pub created: DateTime<Utc>,
pub owner_id: UserId,
Expand All @@ -32,14 +33,15 @@ impl Image {
sqlx::query!(
"
INSERT INTO uploaded_images (
id, url, size, created, owner_id, context, mod_id, version_id, thread_message_id, report_id
id, url, raw_url, size, created, owner_id, context, mod_id, version_id, thread_message_id, report_id
)
VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11
);
",
self.id as ImageId,
self.url,
self.raw_url,
self.size as i64,
self.created,
self.owner_id as UserId,
Expand Down Expand Up @@ -119,7 +121,7 @@ impl Image {
use futures::stream::TryStreamExt;

Check warning on line 121 in src/database/models/image_item.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `futures::stream::TryStreamExt` 

warning: unused import: `futures::stream::TryStreamExt`
   --> src/database/models/image_item.rs:121:13
    |
121 |         use futures::stream::TryStreamExt;
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
sqlx::query!(
"
SELECT id, url, size, created, owner_id, context, mod_id, version_id, thread_message_id, report_id
SELECT id, url, raw_url, size, created, owner_id, context, mod_id, version_id, thread_message_id, report_id
FROM uploaded_images
WHERE context = $1
AND (mod_id = $2 OR ($2 IS NULL AND mod_id IS NULL))
Expand All @@ -142,6 +144,7 @@ impl Image {
Image {
id,
url: row.url,
raw_url: row.raw_url,
size: row.size as u64,
created: row.created,
owner_id: UserId(row.owner_id),
Expand Down Expand Up @@ -185,7 +188,7 @@ impl Image {
|image_ids| async move {
let images = sqlx::query!(
"
SELECT id, url, size, created, owner_id, context, mod_id, version_id, thread_message_id, report_id
SELECT id, url, raw_url, size, created, owner_id, context, mod_id, version_id, thread_message_id, report_id
FROM uploaded_images
WHERE id = ANY($1)
GROUP BY id;
Expand All @@ -197,6 +200,7 @@ impl Image {
let img = Image {
id: ImageId(i.id),
url: i.url,
raw_url: i.raw_url,
size: i.size as u64,
created: i.created,
owner_id: UserId(i.owner_id),
Expand Down
14 changes: 10 additions & 4 deletions src/database/models/oauth_client_item.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ pub struct OAuthClient {
pub id: OAuthClientId,
pub name: String,
pub icon_url: Option<String>,
pub raw_icon_url: Option<String>,
pub max_scopes: Scopes,
pub secret_hash: String,
pub redirect_uris: Vec<OAuthRedirectUri>,
Expand All @@ -31,6 +32,7 @@ struct ClientQueryResult {
id: i64,
name: String,
icon_url: Option<String>,
raw_icon_url: Option<String>,
max_scopes: i64,
secret_hash: String,
created: DateTime<Utc>,
Expand All @@ -53,6 +55,7 @@ macro_rules! select_clients_with_predicate {
clients.id as "id!",
clients.name as "name!",
clients.icon_url as "icon_url?",
clients.raw_icon_url as "raw_icon_url?",
clients.max_scopes as "max_scopes!",
clients.secret_hash as "secret_hash!",
clients.created as "created!",
Expand Down Expand Up @@ -133,15 +136,16 @@ impl OAuthClient {
sqlx::query!(
"
INSERT INTO oauth_clients (
id, name, icon_url, max_scopes, secret_hash, created_by
id, name, icon_url, raw_icon_url, max_scopes, secret_hash, created_by
)
VALUES (
$1, $2, $3, $4, $5, $6
$1, $2, $3, $4, $5, $6, $7
)
",
self.id.0,
self.name,
self.icon_url,
self.raw_icon_url,
self.max_scopes.to_postgres(),
self.secret_hash,
self.created_by.0
Expand All @@ -161,11 +165,12 @@ impl OAuthClient {
sqlx::query!(
"
UPDATE oauth_clients
SET name = $1, icon_url = $2, max_scopes = $3, url = $4, description = $5
WHERE (id = $6)
SET name = $1, icon_url = $2, raw_icon_url = $3, max_scopes = $4, url = $5, description = $6
WHERE (id = $7)
",
self.name,
self.icon_url,
self.raw_icon_url,
self.max_scopes.to_postgres(),
self.url,
self.description,
Expand Down Expand Up @@ -243,6 +248,7 @@ impl From<ClientQueryResult> for OAuthClient {
id: OAuthClientId(r.id),
name: r.name,
icon_url: r.icon_url,
raw_icon_url: r.raw_icon_url,
max_scopes: Scopes::from_postgres(r.max_scopes),
secret_hash: r.secret_hash,
redirect_uris: redirects,
Expand Down
12 changes: 8 additions & 4 deletions src/database/models/organization_item.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ pub struct Organization {

/// The display icon for the organization
pub icon_url: Option<String>,
pub raw_icon_url: Option<String>,
pub color: Option<u32>,
}

Expand All @@ -40,15 +41,16 @@ impl Organization {
) -> Result<(), super::DatabaseError> {
sqlx::query!(
"
INSERT INTO organizations (id, slug, name, team_id, description, icon_url, color)
VALUES ($1, $2, $3, $4, $5, $6, $7)
INSERT INTO organizations (id, slug, name, team_id, description, icon_url, raw_icon_url, color)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
",
self.id.0,
self.slug,
self.name,
self.team_id as TeamId,
self.description,
self.icon_url,
self.raw_icon_url,
self.color.map(|x| x as i32),
)
.execute(&mut **transaction)
Expand Down Expand Up @@ -125,7 +127,7 @@ impl Organization {

let organizations = sqlx::query!(
"
SELECT o.id, o.slug, o.name, o.team_id, o.description, o.icon_url, o.color
SELECT o.id, o.slug, o.name, o.team_id, o.description, o.icon_url, o.raw_icon_url, o.color
FROM organizations o
WHERE o.id = ANY($1) OR LOWER(o.slug) = ANY($2)
GROUP BY o.id;
Expand All @@ -142,6 +144,7 @@ impl Organization {
team_id: TeamId(m.team_id),
description: m.description,
icon_url: m.icon_url,
raw_icon_url: m.raw_icon_url,
color: m.color.map(|x| x as u32),
};

Expand All @@ -168,7 +171,7 @@ impl Organization {
{
let result = sqlx::query!(
"
SELECT o.id, o.slug, o.name, o.team_id, o.description, o.icon_url, o.color
SELECT o.id, o.slug, o.name, o.team_id, o.description, o.icon_url, o.raw_icon_url, o.color
FROM organizations o
LEFT JOIN mods m ON m.organization_id = o.id
WHERE m.id = $1
Expand All @@ -187,6 +190,7 @@ impl Organization {
team_id: TeamId(result.team_id),
description: result.description,
icon_url: result.icon_url,
raw_icon_url: result.raw_icon_url,
color: result.color.map(|x| x as u32),
}))
} else {
Expand Down
Loading

0 comments on commit 82de2ac

Please sign in to comment.