diff --git a/.dockerignore b/.dockerignore index bfba80457c4..c17f59316f6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,6 @@ public/assets public/uploads public/uploads-avatar public/uploads-replay -resources/assets/build/ +resources/builds/ storage/ vendor/ diff --git a/.env.dusk.local.example b/.env.dusk.local.example index f4da15ef15f..e73eadd68fb 100644 --- a/.env.dusk.local.example +++ b/.env.dusk.local.example @@ -1,8 +1,8 @@ APP_KEY= APP_ENV=testing -APP_URL=http://nginx -NOTIFICATION_ENDPOINT=/home/notifications/feed-dusk +APP_URL=http://localhost:8000 +NOTIFICATION_ENDPOINT=ws://notification-server-dusk:2345 DB_DATABASE=osu_test DB_DATABASE_CHAT=osu_chat_test @@ -11,5 +11,3 @@ DB_DATABASE_STORE=osu_store_test DB_DATABASE_UPDATES=osu_updates_test ES_INDEX_PREFIX=test_ - -CACHE_DRIVER_LOCAL=array diff --git a/.env.example b/.env.example index 19c7b8583ff..2778316755f 100644 --- a/.env.example +++ b/.env.example @@ -36,7 +36,6 @@ OSU_API_KEY= BROADCAST_DRIVER=log CACHE_DRIVER=file -# CACHE_DRIVER_LOCAL=file SESSION_DRIVER=file # SESSION_DOMAIN= # SESSION_SECURE_COOKIE=false @@ -230,6 +229,7 @@ CLIENT_CHECK_VERSION=false # USER_MAX_SCORE_PINS=10 # USER_MAX_SCORE_PINS_SUPPORTER=50 +# USER_HIDE_PINNED_SOLO_SCORES=true # the content is in markdown format # USER_PROFILE_SCORES_NOTICE= @@ -251,8 +251,16 @@ CLIENT_CHECK_VERSION=false # OAUTH_MAX_USER_CLIENTS=1 -# USER_REPORT_NOTIFICATION_ENDPOINT_MODERATION= # USER_REPORT_NOTIFICATION_ENDPOINT_CHEATING= +# default if nothing specified for specific type +# USER_REPORT_NOTIFICATION_ENDPOINT_MODERATION= + +# USER_REPORT_NOTIFICATION_ENDPOINT_BEATMAPSET= +# USER_REPORT_NOTIFICATION_ENDPOINT_BEATMAPSET_DISCUSSION= +# USER_REPORT_NOTIFICATION_ENDPOINT_CHAT= +# USER_REPORT_NOTIFICATION_ENDPOINT_COMMENT= +# USER_REPORT_NOTIFICATION_ENDPOINT_FORUM= +# USER_REPORT_NOTIFICATION_ENDPOINT_USER= # LOG_CHANNEL=single @@ -281,6 +289,8 @@ CLIENT_CHECK_VERSION=false # TWITCH_CLIENT_SECRET= # SCORES_ES_CACHE_DURATION= +# SCORES_EXPERIMENTAL_RANK_AS_DEFAULT=false +# SCORES_EXPERIMENTAL_RANK_AS_EXTRA=false # SCORES_RANK_CACHE_LOCAL_SERVER=0 # SCORES_RANK_CACHE_MIN_USERS=35000 # SCORES_RANK_CACHE_SERVER_URL= @@ -293,3 +303,13 @@ CLIENT_CHECK_VERSION=false # TRUSTED_PROXIES= # IS_DEVELOPMENT_DEPLOY=true +# OSU_EXPERIMENTAL_HOST= + +# OSU_URL_LAZER_ANDROID='https://github.com/ppy/osu/releases/latest/download/sh.ppy.osulazer.apk' +# OSU_URL_LAZER_IOS='/home/testflight' +# OSU_URL_LAZER_LINUX_X64='https://github.com/ppy/osu/releases/latest/download/osu.AppImage' +# OSU_URL_LAZER_MACOS_AS='https://github.com/ppy/osu/releases/latest/download/osu.app.Apple.Silicon.zip' +# OSU_URL_LAZER_OTHER='https://github.com/ppy/osu/#running-osu' +# OSU_URL_LAZER_WINDOWS_X64='https://github.com/ppy/osu/releases/latest/download/install.exe' +# OSU_URL_LAZER_INFO= +# OSU_URL_USER_RESTRICTION=/wiki/Help_centre/Account_restrictions diff --git a/.env.testing.example b/.env.testing.example index 2dbcf456d24..1e15950708d 100644 --- a/.env.testing.example +++ b/.env.testing.example @@ -1,10 +1,12 @@ -DB_DATABASE=osu_test DB_DATABASE_CHAT=osu_chat_test DB_DATABASE_MP=osu_mp_test DB_DATABASE_STORE=osu_store_test DB_DATABASE_UPDATES=osu_updates_test DB_DATABASE_CHARTS=osu_charts_test +# match with docker-compose.yml +DB_DATABASE=osu_test ES_INDEX_PREFIX=test_ +SCHEMA=test PAYMENT_SANDBOX=true diff --git a/.eslintrc.js b/.eslintrc.js index a8aba92469a..d728cea6d14 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -17,7 +17,7 @@ module.exports = { 'plugin:react-hooks/recommended', 'plugin:react/recommended', ], - files: ['resources/assets/lib/**/*.{ts,tsx}', 'tests/karma/**/*.ts'], + files: ['resources/js/**/*.{ts,tsx}', 'tests/karma/**/*.{ts,tsx}'], parser: '@typescript-eslint/parser', plugins: [ '@typescript-eslint', @@ -170,7 +170,7 @@ module.exports = { browser: true, node: false, }, - files: ['resources/assets/lib/**/*.{ts,tsx}'], + files: ['resources/js/**/*.{ts,tsx}'], parserOptions: { project: 'tsconfig.json', sourceType: 'module', @@ -181,7 +181,7 @@ module.exports = { browser: false, node: true, }, - files: ['tests/karma/**/*.ts'], + files: ['tests/karma/**/*.{ts,tsx}'], parserOptions: { project: 'tests/karma/tsconfig.json', sourceType: 'module', diff --git a/.gitattributes b/.gitattributes index 4dee8d23318..4c887056fcf 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,4 @@ * text=auto *.sh eol=lf /public/images/** binary -/resources/assets/fonts/** binary +/resources/fonts/** binary diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ae46e599382..986efc213d2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,14 +27,14 @@ jobs: uses: shivammathur/setup-php@v2 with: tools: composer:v2 - php-version: '8.0' + php-version: '8.1' - name: Get composer cache directory id: composercache - run: echo "::set-output name=dir::$(composer config cache-files-dir)" + run: echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT - name: Cache composer - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ steps.composercache.outputs.dir }} key: ${{ runner.os }}-composer-${{ hashFiles('composer.lock') }} @@ -48,7 +48,7 @@ jobs: - name: Install js dependencies run: yarn --frozen-lockfile - - run: 'yarn lint --max-warnings 177 > /dev/null' + - run: 'yarn lint --max-warnings 123 > /dev/null' - run: ./bin/update_licence.sh -nf diff --git a/.github/workflows/pack.yml b/.github/workflows/pack.yml index 063c27f44b8..baf5116a20d 100644 --- a/.github/workflows/pack.yml +++ b/.github/workflows/pack.yml @@ -57,11 +57,47 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + notify_pending_production_deploy: + if: ${{ github.ref_type == 'tag' }} + runs-on: ubuntu-latest + needs: + - push_to_registry + steps: + - + name: Submit pending deployment notification + run: | + export TITLE="Pending osu-web Production Deployment: $GITHUB_REF_NAME" + export URL="https://github.com/ppy/osu-web/actions/runs/$GITHUB_RUN_ID" + export DESCRIPTION="Docker image was built for tag $GITHUB_REF_NAME and awaiting approval for production deployment: + [View Workflow Run]($URL)" + export ACTOR_ICON="https://avatars.githubusercontent.com/u/$GITHUB_ACTOR_ID" + + BODY="$(jq --null-input '{ + "embeds": [ + { + "title": env.TITLE, + "color": 15098112, + "description": env.DESCRIPTION, + "url": env.URL, + "author": { + "name": env.GITHUB_ACTOR, + "icon_url": env.ACTOR_ICON + } + } + ] + }')" + + curl \ + -H "Content-Type: application/json" \ + -d "$BODY" \ + "${{ secrets.DISCORD_INFRA_WEBHOOK_URL }}" + push_to_production: if: ${{ github.ref_type == 'tag' }} runs-on: ubuntu-latest needs: - push_to_registry + environment: production steps: - name: Checkout @@ -73,6 +109,7 @@ jobs: token: ${{ secrets.KUBERNETES_CONFIG_REPO_ACCESS_TOKEN }} repository: ppy/osu-kubernetes-config event-type: osu-web-deploy + client-payload: '{ "dockerTag": "${{ github.ref_name }}" }' - name: Create Sentry release uses: getsentry/action-release@v1 @@ -90,6 +127,7 @@ jobs: runs-on: ubuntu-latest needs: - push_to_registry + environment: staging steps: - name: Checkout diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2a68bc91a87..a0b57880fe4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,40 +10,81 @@ env: APP_KEY: base64:q7U5qyAkedR1F6UhN0SQlUxBpAMDyfHy3NNFkqmiMqA= APP_URL: http://localhost:8000 CACHE_DRIVER: redis - CACHE_DRIVER_LOCAL: array DB_HOST: 127.0.0.1 - DB_USERNAME: root ES_SOLO_SCORES_HOST: http://localhost:9200 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NOTIFICATION_ENDPOINT: ws://127.0.0.1:2345 - NOTIFICATION_REDIS_HOST: 127.0.0.1 OSU_INSTALL_DEV: 1 OSU_USE_SYSTEM_COMPOSER: 1 OSU_DB_CREATE: 1 PAYMENT_SANDBOX: true PASSPORT_PRIVATE_KEY: |+ -----BEGIN PRIVATE KEY----- - MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAMevwZweM2xj5GFv - FY0lzE4b/IQEIBO0NkMBcv1E2zQD7l4cGEft//ksZrP2l5y0l+FCL/2178bt+gZg - oXalVTrjQYhtP1B3yngpu+N7O75JT+xabMLEsCEnsuFyInfTl+kUGciqX8fIdELx - 9Tgx5nwFkNqhE5Yo3fkhRjnjMaHhAgMBAAECgYEAsIrFvFXDFwAVyMKiJiEVyLTb - gof3KBR6qLDeTeaTeiBDnPZvzSAw38YOourPzd6oLKIMtd0lORXqp7rE5ZV0jEOo - QhDuahbMwfTeDOihhTTyc0ZxtY4WlVuV3lbNLZYSKXwpNU/Dud9mWyqSnS5+QcrF - c/Z6mxpVS6cxBFxrAaUCQQD0LHfl9R3aZbRPpDRX61eOi2wT96dZVi0IV+Yq4qqJ - L/ZDxCpTBx5xMvVqmET7Q69KULzUfajHOgkW3h+JA8vTAkEA0VuwDMcF6N4TGrpE - 7rwLdFrXVopDCf1IT+y0spf52RUe7dGsUPz+Ed1oQ55S+ZPjAiHIF/EDGhtGkhbc - h5fO+wJATVe0ltOgpCgpCD0UE3FJZ66ECoMcsDCazRTCpHzt1cyqbea6HViY7PRe - Rmh7GTfkU17loU04Y/rfHC45wPXB3wJARE7dHx4kwHkM1LCn4xj5x8oH7pWIEA9S - 87jwsEDD5V9tyvyZc4dIQn3yEfXrbsEg1UY+aglaEL+LhAjvhVIz1wJBANGE8xeu - qLUIE7/3nfacXiK/v02qKlW6++LrvSzgAOLsYyqfYpfmKffSdkTxFIZ8wUjufuOT - DPxkn/u5C+BuAqQ= + MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCmEdH3wybDftSh + DkAPHA/43mAelOj135qhqEWUvM3vz2ggVMT9WT9A+fZiFZzZc0K9GCzKlpnSRlb5 + xycwtoUCUL/vOxts2cpPu/OVTiyDrF8+4QtjuVAMTugFdcatUTuQXbxawbNGBazk + M3+BGd1nSAuNMwEY6od6kEE1OGS/ePkGctZxfh/WOmFeT9GtQLyMSab478wjPS4+ + vAyjgHHuBOYqZsFbu/WUURmAa6qTYGoZ7x21R00vP7fjnHahzezidDwAe6s9XX0N + ieh5fGypsgOQSpwiFXiENTObL8RSKhFsN4rWYv7JtGpOwF6uTf8ioj/9iQ2G2Wa0 + zegdzLaprKZxLBMA0XqQOG2lWzRujW8zvvjrWDo1WxAz5JvKzONcHU0o7vbgD07u + Fsi1QJCJ8sTFzq/GGqKA1jc+RehDbfg2iXm83V8ioutmpd9Cp3wmwqZsv84sCtVn + lH6DE094j9p6A25oi++wr3M2XIxs51CohTstMUYdnlsdxLPeFp+0v50jU1WfqM5e + kFExZYuPI+ukUxvrJ99BhOAu0WMlQRPpI1qxeIkmH3TLXJIrC/7kzYnyqkUbKeiX + PZcmr5WC7PWOEiVGaDo762FNlzvV+DS9xSNwu4/LbPtU1fhhw+mSZ6/rw08JakIC + qLTOMzIdBl2H9JwvjSgMZvodF0pqfQIDAQABAoICAElhsHUZr6RNA+nS4S2MCea/ + ZJu+0x3qW2q5b12k/B7OeEVpD86fRBsTGPMId9GCY+goqYovd/L5j+85ODEHRcjb + I6Szao/zwLjw5VaBP5xDa6beTGkZdqyppU7cVxk4vugw80zrMKttUJNZyiqi4jmE + FU6kTgTThV/8JEQ/Eg2lh+x4jBeEUs8X5vSEsrMq2uXmmbiARaUAoNGpyK/qzu0N + DyOHWkONz5tlJq6fZLVQsjGZfnge9JU/Q6N4/NbMprL+B5FzFBy+lcaIvMgJ2f42 + Ier9fPZ2pFVPKOmW9toTqrU16cyH2+wHI75+tJey8V09cqc2i1qd8AtOvRWF4uRq + hhUdbb4loGaR8J4C3oOLUb86Bv1lNeC1omVW+K7b/1DAcZinEe4evZU5sbQs4FB1 + XopVIPAZBw0JZU5clJ3jteWo1roa6U8KkcX+gxXMCD+qlyOX6YUjRvQdXawTBh7M + 3tsdOvHWt4htrPZMNUqK5Qo86ASSVxiWNZlTSZhK+as6DHNH+6iXw4NruBV5hROE + 1Rpoh+kIRQ193XX/kquwMpnsITT+y7KGWUGyxa2S4aK3LQOsekQfn0ysVaIMuJ99 + kNH5jCWq9ituRnqH78JafIr/l9V3VQtax2ot9GTk7lJ+Ru4eUoIktM6AV1KrqBbO + wWU5TRt2JWJ2McoYJsvFAoIBAQDVwIMnOYaepJMh2qUs25tZA73E9NB8PKXlzuwf + GyyZ7OKFtCMBT1UnCOZk3ha1qSOcW5zatX2nwKefqM+sn6MQXE0Yh1yeMWg9KUtu + 3U9k191YNlQLGbMIzIzAksc+ceyIH/Ma3jaw8LWVTItWljSGEGL/C2xclqSVUtSU + k87SOFedOBsF5PUxPFetolUU1wSVjRXEOFqxegRwJfc/Yjm7DM3UEqAYVRkcDMqK + 4gYtlTyAPmG7HIy3GD6EvRT/dJ7D4pJb5UYEsDPn1h/+cSKzRO+1282Pxja/n4VB + WpWVMuFYMD1T1fFXq6pTb17WqtTMIdXpjTM7KaPLTeIRZryPAoIBAQDG5Kj0p97b + e3Aaf2BLg71u4G2SWvR7lkGAa0nmEa9sIUxuKUQV/tY1xvaGbIRDxbLKFVLAe1Zf + SE3ziWLwj+HXXYq0UEsjzlpVWAUzVpmjdTvSNp3dFeGR/Fe1d/gLQ/zJGYOKJE0I + acAH5tNAxQlE+a6tbOeb7RtEDBCReRL73MRl2y+jFVe84nPNVrAZxKE9l447+9oU + w8tT1lIm8/5WHuu51HTGD73k5qwfOt5PpwCZLkxHG+RiSbom4bQajqFT5tA7rmB9 + CRGVbrAwD42ZebJ1sZXUc6PmwCDlrwl+krMtRvpDcaD+qYRxnTVF/0g82RoqFZUZ + 6eTuD2Vk4IYzAoIBAQCwrZE6qMfm9S1QPENvEo4TQ45l790r841EqIZvJ6z0BeMu + lLiiaop921NkaFXpDccLhIXgGUelkw56X7RYgRLtgP2QmsIpV7lLZIifOHpZZjvk + n/NTwYhEa56jxi6+JLhXw+DTgn0+P5g3sa8ygLElZwIMwcpttW6QpohjztCS3lM8 + 9pUPiDJK4g+uy5D7ysZVPuqQ5+u+6pYkhJ0bXPtO6sRu2H5P3ncEwmXf6fclUkdR + 0T5CxNRiur1iRk/G3wQj8XD8WERJt8MPi+OiEq2V9BMXBHmirzmtphMk89TUH9w3 + YpguZZY42wHMOS/wy/9cQ1Nm2nlQG5jxgIytyXdtAoIBAQCQ1XpjjssusRjVcuRA + mVKdJzkjVrCqPqS8S2So8T/5UdZCcjJuggCjcJudD/DRGWo01opIOIiRIioBqo4t + 99jR6ABVhhLxpBcMBujilbGT4o8JaVRT5tc4Y7XLI+2w5nVyS+4J7p1Z/wgjuOcl + R1aUr+2PxLp0RZmRV+fIx44XcpBiS9bIv4YAujbx1KqwWQvBuleNPr10WY/7IRr5 + 2rvfIu0tiZTvlg8GXSQCKfAxuL8qzRgmwBzzphS0s69fl3XLj00b6MPCyZPQaaZL + mUfX91MJeaRN7VfxdP9/wpL7ZQCte/5KlrhSOkeEJKLish98j+wJsWW+VSCsavG/ + H/EZAoIBAGfvtsP++9a6YP80lZ5uadP9WH3D74oMelbyaFRWVK4wqMqFqet1aSJJ + 19S6fRlkpTm2RtmJhleOa+4pbNO1KQRd1ZGDl0m3UN+NNwS3TRGzOlXUPXnxcwRU + TemRmKnjEce6vC/Gh0E+5CdTjifNmce2MPyGgvGAV3GIgSAM2yIqKCjWo96eSThl + 5QttXhvWxNP+RhLUkxpd9nEIilf4p0etUnjkTeHcDw67KjfS0YnJ9VGq1iCLRlGh + suKs9738LlFwEEDFD73iRY7o/Wz9wox7E0CuqfkAikwYHiGsI2n+bL7AXNj8k4GF + qkWREaw6Mt2MfHwSFoJooagIuAIHMGw= -----END PRIVATE KEY----- PASSPORT_PUBLIC_KEY: |+ -----BEGIN PUBLIC KEY----- - MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDHr8GcHjNsY+RhbxWNJcxOG/yE - BCATtDZDAXL9RNs0A+5eHBhH7f/5LGaz9pectJfhQi/9te/G7foGYKF2pVU640GI - bT9Qd8p4Kbvjezu+SU/sWmzCxLAhJ7LhciJ305fpFBnIql/HyHRC8fU4MeZ8BZDa - oROWKN35IUY54zGh4QIDAQAB + MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAphHR98Mmw37UoQ5ADxwP + +N5gHpTo9d+aoahFlLzN789oIFTE/Vk/QPn2YhWc2XNCvRgsypaZ0kZW+ccnMLaF + AlC/7zsbbNnKT7vzlU4sg6xfPuELY7lQDE7oBXXGrVE7kF28WsGzRgWs5DN/gRnd + Z0gLjTMBGOqHepBBNThkv3j5BnLWcX4f1jphXk/RrUC8jEmm+O/MIz0uPrwMo4Bx + 7gTmKmbBW7v1lFEZgGuqk2BqGe8dtUdNLz+345x2oc3s4nQ8AHurPV19DYnoeXxs + qbIDkEqcIhV4hDUzmy/EUioRbDeK1mL+ybRqTsBerk3/IqI//YkNhtlmtM3oHcy2 + qaymcSwTANF6kDhtpVs0bo1vM77461g6NVsQM+SbyszjXB1NKO724A9O7hbItUCQ + ifLExc6vxhqigNY3PkXoQ234Nol5vN1fIqLrZqXfQqd8JsKmbL/OLArVZ5R+gxNP + eI/aegNuaIvvsK9zNlyMbOdQqIU7LTFGHZ5bHcSz3haftL+dI1NVn6jOXpBRMWWL + jyPrpFMb6yffQYTgLtFjJUET6SNasXiJJh90y1ySKwv+5M2J8qpFGynolz2XJq+V + guz1jhIlRmg6O+thTZc71fg0vcUjcLuPy2z7VNX4YcPpkmev68NPCWpCAqi0zjMy + HQZdh/ScL40oDGb6HRdKan0CAwEAAQ== -----END PUBLIC KEY----- REDIS_HOST: 127.0.0.1 SESSION_DRIVER: redis @@ -55,66 +96,23 @@ jobs: tests: strategy: matrix: - php: ['8.0'] + php: ['8.1'] name: Tests runs-on: ubuntu-latest - services: - db: - image: mysql:8.0 - ports: - - 3306:3306 - options: >- - --health-cmd="mysqladmin ping" - --health-interval=10s - --health-timeout=5s - --health-retries=5 - -e MYSQL_ALLOW_EMPTY_PASSWORD=1 - --entrypoint sh mysql:8.0 -c "exec docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password" - - elasticsearch: - env: - action.auto_create_index: "false" - discovery.type: single-node - image: elasticsearch:7.17.6 - ports: - - 9200:9200 - - redis: - image: redis - ports: - - 6379:6379 - - osu-beatmap-difficulty-lookup-cache: - image: pppy/osu-beatmap-difficulty-lookup-cache - ports: - - 5000:80 - - osu-elastic-indexer: - image: pppy/osu-elastic-indexer - options: >- - -e DB_CONNECTION_STRING=Server=db;Database=osu;Uid=root - -e ES_HOST=http://elasticsearch:9200 - -e REDIS_HOST=redis - -e SCHEMA=test - --entrypoint sh pppy/osu-elastic-indexer -c "exec dotnet osu.ElasticIndexer.dll queue --force-version --wait" - - osu-notification-server: - env: - APP_KEY: ${{ env.APP_KEY }} - DB_HOST: db - DB_USERNAME: ${{ env.DB_USERNAME }} - NOTIFICATION_REDIS_HOST: redis - NOTIFICATION_SERVER_LISTEN_HOST: 0.0.0.0 - PASSPORT_PUBLIC_KEY: ${{ env.PASSPORT_PUBLIC_KEY }} - REDIS_HOST: redis - image: pppy/osu-notification-server - ports: - - 2345:2345 - steps: - name: Checkout uses: actions/checkout@v3 + - name: Services + run: 'docker compose up --quiet-pull --wait + beatmap-difficulty-lookup-cache + db + elasticsearch + notification-server + redis + score-indexer + ' + - name: Setup node.js uses: actions/setup-node@v3 with: @@ -124,16 +122,16 @@ jobs: - name: Set php version uses: shivammathur/setup-php@v2 with: - extensions: redis + extensions: redis, swoole tools: composer:v2 php-version: ${{ matrix.php }} - name: Get composer cache directory id: composercache - run: echo "::set-output name=dir::$(composer config cache-files-dir)" + run: echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT - name: Cache composer - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ steps.composercache.outputs.dir }} key: ${{ runner.os }}-composer-${{ hashFiles('composer.lock') }} @@ -150,15 +148,23 @@ jobs: - name: Generate docs run: php artisan scribe:generate - - name: Run karma - run: yarn karma start --single-run --browsers ChromeHeadless - - name: Run PHPUnit run: ./bin/phpunit.sh + # TODO: workaround things (beatmaps) being indexed during test above and not cleaned up. + # This used to cause beatmap listing returning cursor with Long.MIN_VALUE for null timetamp + # and errors out when trying to get the next page (es can't parse such value for timestamp) + # but has since been fixed. + # Something should still be done regarding es index between tests though. + - name: Clean indexes + run: php artisan es:index-documents --yes + - name: Run Dusk run: ./bin/run_dusk.sh + - name: Run karma + run: yarn karma start --single-run --browsers ChromeHeadless + # this only tests that the rollback functions are valid and doesn't check # if they actually do what they're expected to do. - name: Migration rollback test diff --git a/.gitignore b/.gitignore index d63dc39ee41..ac93624bd8b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Local configuration. /.env* !/.env*.example +/.webpack-build-notifier-config.js # ignore _ide_helper.php _ide_helper.php @@ -20,12 +21,12 @@ desktop.ini /yarn-error.log # Compiled/copied assets +/.scribe /public/assets /public/docs /public/uploads /public/uploads-avatar /public/uploads-replay -/resources/docs # server logs /logs/osu/* @@ -68,7 +69,7 @@ com_crashlytics_export_strings.xml sftp-settings.json -/resources/assets/build/ +/resources/builds/ composer-installer @@ -81,3 +82,6 @@ composer-installer /bootstrap/cache/ /storage/htmlpurifier/ /storage/paypal_auth.cache + +/database/ip2asn/*.idx +/database/ip2asn/*.tsv diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 33b6fb8ebca..e1bc0bc1f6c 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -26,7 +26,7 @@ Note that as the actual process is run as non-root user, the files must be world ## Services -The image built serves multiple purposes, each can be run by passing the parameter when starting docker (or as command if using orchestration tools like docker-compose). +The image built serves multiple purposes, each can be run by passing the parameter when starting docker (or as command if using orchestration tools like docker compose). There are three main commands: diff --git a/Dockerfile.deployment b/Dockerfile.deployment index cc1920db21c..ced5b8e6345 100644 --- a/Dockerfile.deployment +++ b/Dockerfile.deployment @@ -1,40 +1,40 @@ -FROM ubuntu:20.04 +FROM debian:11 -RUN apt-get update -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common +RUN apt update +RUN DEBIAN_FRONTEND=noninteractive apt install -y apt-transport-https lsb-release ca-certificates curl -RUN add-apt-repository ppa:ondrej/php +RUN curl -sSLo /usr/share/keyrings/deb.sury.org-php.gpg https://packages.sury.org/php/apt.gpg +RUN echo "deb [signed-by=/usr/share/keyrings/deb.sury.org-php.gpg] https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list -RUN apt-get update -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y \ +RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - + +RUN apt update +RUN DEBIAN_FRONTEND=noninteractive apt install -y \ build-essential \ - curl \ git \ jhead \ nginx \ - php8.0-common \ - php8.0-curl \ - php8.0-ds \ - php8.0-gd \ - php8.0-intl \ - php8.0-mbstring \ - php8.0-mysql \ - php8.0-redis \ - php8.0-sqlite3 \ - php8.0-swoole \ - php8.0-tokenizer \ - php8.0-xml \ - php8.0-zip \ - php8.0 \ + nodejs \ + php8.1-common \ + php8.1-curl \ + php8.1-ds \ + php8.1-gd \ + php8.1-intl \ + php8.1-mbstring \ + php8.1-mysql \ + php8.1-redis \ + php8.1-sqlite3 \ + php8.1-swoole \ + php8.1-tokenizer \ + php8.1-xml \ + php8.1-zip \ + php8.1 \ zip -WORKDIR /app - -RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - -RUN apt-get update -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs RUN npm install -g yarn +WORKDIR /app + RUN curl -L "https://getcomposer.org/download/latest-2.x/composer.phar" > /usr/local/bin/composer && chmod 755 /usr/local/bin/composer COPY composer.json composer.lock ./ @@ -55,6 +55,7 @@ ARG APP_URL ARG DOCS_URL RUN yarn production +RUN php artisan ip2asn:update RUN php artisan scribe:generate RUN rm -rf node_modules diff --git a/Dockerfile.development b/Dockerfile.development index 06b3515a09d..cebf8527133 100644 --- a/Dockerfile.development +++ b/Dockerfile.development @@ -1,52 +1,50 @@ -FROM ubuntu:20.04 +FROM debian:11 -RUN apt-get update -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common +RUN apt update +RUN DEBIAN_FRONTEND=noninteractive apt install -y apt-transport-https lsb-release ca-certificates curl -RUN add-apt-repository ppa:ondrej/php +RUN curl -sSLo /usr/share/keyrings/deb.sury.org-php.gpg https://packages.sury.org/php/apt.gpg +RUN echo "deb [signed-by=/usr/share/keyrings/deb.sury.org-php.gpg] https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list -RUN apt-get update -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y \ +RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - + +RUN apt update +RUN DEBIAN_FRONTEND=noninteractive apt install -y \ build-essential \ - curl \ + chromium-driver \ + default-mysql-client \ git \ gosu \ jhead \ libglib2.0-0 \ libnss3 \ - mysql-client \ netcat-openbsd \ - php8.0-common \ - php8.0-curl \ - php8.0-ds \ - php8.0-fpm \ - php8.0-gd \ - php8.0-intl \ - php8.0-mbstring \ - php8.0-mysql \ - php8.0-redis \ - php8.0-sqlite3 \ - php8.0-swoole \ - php8.0-tokenizer \ - php8.0-xml \ - php8.0-zip \ - php8.0 \ + nodejs \ + php8.1 \ + php8.1-common \ + php8.1-curl \ + php8.1-ds \ + php8.1-gd \ + php8.1-intl \ + php8.1-mbstring \ + php8.1-mysql \ + php8.1-redis \ + php8.1-sqlite3 \ + php8.1-swoole \ + php8.1-tokenizer \ + php8.1-xml \ + php8.1-zip \ zip -RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - -RUN apt-get update -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs RUN npm install -g yarn RUN curl -L "https://getcomposer.org/download/latest-2.x/composer.phar" > /usr/local/bin/composer && chmod 755 /usr/local/bin/composer - -COPY install_chrome.sh ./ - -RUN ./install_chrome.sh +RUN mv /usr/bin/chromium /usr/bin/chromium.orig +COPY chromium /usr/bin/ WORKDIR /app RUN groupadd osuweb && useradd -g osuweb osuweb ENTRYPOINT ["/app/docker/development/entrypoint.sh"] -CMD ["serve"] +CMD ["octane"] diff --git a/README.md b/README.md index c256d5d11bf..339f897f82c 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The browser-facing portion of [osu!](https://osu.ppy.sh/home). ## Requirements -- A PHP 8.0+ environment +- A PHP 8.1+ environment - MySQL 8.0+ - Elasticsearch @@ -27,7 +27,7 @@ Please see [CONTRIBUTING.md](CONTRIBUTING.md) for information about the code sta While we have standards in place, nothing is set in stone. If you have an issue with the way code is structured; with any libraries we are using; with any processes involved with contributing, *please* bring it up. We welcome all feedback so we can make contributing to this project as pain-free as possible. -For those interested, we love to reward quality contributions via [bounties](https://docs.google.com/spreadsheets/d/1jNXfj_S3Pb5PErA-czDdC9DUu4IgUbe1Lt8E7CYUJuE/view?&rm=minimal#gid=523803337), paid out via paypal or osu! supporter tags. Don't hesitate to [request a bounty](https://docs.google.com/forms/d/e/1FAIpQLSet_8iFAgPMG526pBZ2Kic6HSh7XPM3fE8xPcnWNkMzINDdYg/viewform) for your work on this project. +We love to reward quality contributions. If you have made a large contribution or are a regular contributor, you are welcome to [submit an expense via opencollective](https://opencollective.com/ppy/expenses/new). If you have any questions, feel free to [reach out to peppy](mailto:pe@ppy.sh) before doing so. ## Seeking Help diff --git a/SETUP.md b/SETUP.md index 5c278265c4e..7b3812bed68 100644 --- a/SETUP.md +++ b/SETUP.md @@ -68,14 +68,12 @@ At this point you should be able to access the site via whatever webserver you c - Run `bin/docker_dev.sh`. Make sure the repository folder is owned by the user executing this command (must be non-root). - Due to the nature of Docker (a container is killed when the command running in it finishes), the Yarn container will be run in watch mode. - Do note that the supplied Elasticsearch container uses a high (1+ GB) amount of RAM. Ensure that your system (or virtual machine, if running on Windows/macOS) has a necessary amount of memory allocated (at least 2 GB). If you can't (or don't want to), you can comment out the relevant elasticsearch lines in `docker-compose.yml`. -- To run any of the below commands, make sure you are using the docker container: `docker-compose run --rm php`. - - To run artisan commands, run using `docker-compose run --rm php artisan`. +- To run any of the below commands, make sure you are using the docker container: `docker compose run --rm php`. + - To run artisan commands, run using `docker compose run --rm php artisan`. --- **Notes** -Newer versions of docker can use `docker compose` instead of `docker-compose` - The `elasticsearch` and `db` containers store their data to volumes, the containers will use data on these volumes if they already exist. ### Elasticsearch @@ -106,7 +104,7 @@ The Mysql images provided by Docker and Mysql have different uids for the `mysql update the ownership of the mysql data files: - docker-compose run --rm db sh -c 'chown -R mysql:mysql /var/lib/mysql' + docker compose run --rm db sh -c 'chown -R mysql:mysql /var/lib/mysql' ### Windows @@ -122,26 +120,6 @@ git config core.eol lf git config core.filemode false ``` -### ARM-based CPUs - -Tests that require the use of Chrome (both Karma and Dusk tests) will not work inside Docker when running on ARM-based CPUs (e.g. Macs running Apple Silicon). In this scenario, these tests should be run outside of a container. - -Dusk tests can make use of an external Chrome driver instance by setting the following environment variables: -- `DUSK_WEBDRIVER_URL` the url of the Chrome driver accessible from the container. -- `APP_URL` the url that will be used to access the app running in the container from Chrome. - -e.g. If Docker Desktop and the default networking are used and `chromedriver` is running on the host: - - docker-compose run --rm -e DUSK_WEBDRIVER_URL=host.docker.internal:9515 APP_URL=http://127.0.0.1:8080 php test browser - -The host `chromedriver` will need to allow connections from the container: - - chromedriver --whitelisted-ips --allowed-origins='host.docker.internal' - -Other custom configurations to run the tests within the container are currently not supported. - ---- - ### Docker hints #### Services @@ -170,7 +148,7 @@ Sometimes a restart of notification-server and notification-server-dusk will be See if anything has stopped: ``` -docker-compose ps +docker compose ps ``` Start docker in background: @@ -178,19 +156,19 @@ Start docker in background: ``` bin/docker_dev.sh -d # alternatively -# docker-compose up -d +# docker compose up -d ``` Start single docker service: ``` -docker-compose start +docker compose start ``` Restart single docker service: ``` -docker-compose restart +docker compose restart ``` #### Direct database access @@ -200,7 +178,7 @@ Using own mysql client, connect to port 3306 or `MYSQL_EXTERNAL_PORT` if set whe Alternatively, there's mysql client installed in php service: ``` -docker-compose run --rm php mysql +docker compose run --rm php mysql ``` #### Updating image @@ -208,25 +186,25 @@ docker-compose run --rm php mysql Docker images need to be occasionally updated to make sure they're running latest version of the packages. ``` -docker-compose down --rmi all -docker-compose pull -docker-compose build --pull +docker compose down --rmi all +docker compose pull +docker compose build --pull ``` (don't use `build --no-cache` as it'll end up rebuilding `php` image multiple times) #### Faster php commands -When frequently running commands, doing `docker-compose run` may feel a little bit slow. An alternative is by running the command in existing instance instead. For example to run `artisan tinker`: +When frequently running commands, doing `docker compose run` may feel a little bit slow. An alternative is by running the command in existing instance instead. For example to run `artisan tinker`: ``` -docker-compose exec php /app/docker/development/entrypoint.sh artisan tinker +docker compose exec php /app/docker/development/entrypoint.sh artisan tinker ``` -Add an alias for the docker-compose command so it doesn't need to be specified every time: +Add an alias for the docker compose command so it doesn't need to be specified every time: ``` -alias p='docker-compose exec php /app/docker/development/entrypoint.sh' +alias p='docker compose exec php /app/docker/development/entrypoint.sh' p artisan tinker ``` @@ -252,7 +230,7 @@ Using Laravel's [Mix](https://laravel.com/docs/6.x/mix). $ yarn run development ``` -Note that if you use the bundled docker-compose setup, yarn/webpack will be already run in watch mode. +Note that if you use the bundled docker compose setup, yarn/webpack will be already run in watch mode. ## Reset the database + seeding sample data @@ -260,7 +238,7 @@ Note that if you use the bundled docker-compose setup, yarn/webpack will be alre $ php artisan migrate:fresh --seed ``` -Run the above command to rebuild the database and seed with sample data. In order for the seeder to seed beatmaps, you must enter a valid osu! API key as the value of the `OSU_API_KEY` property in the `.env` configuration file, as the seeder obtains beatmap data from the osu! API. The key can be obtained at [the "osu! API Access" page](https://old.ppy.sh/p/api), which is currently only available on the old site. +Run the above command to rebuild the database and seed with sample data. In order for the seeder to seed beatmaps, you must enter a valid osu! API key as the value of the `OSU_API_KEY` property in the `.env` configuration file, as the seeder obtains beatmap data from the osu! API. The key can be obtained from [the "Legacy API" section of your account settings page](https://osu.ppy.sh/home/account/edit#legacy-api). ## Continuous asset generation while developing @@ -300,7 +278,7 @@ APP_ENV=testing php artisan migrate:fresh --yes or if using docker: ``` -docker-compose run --rm -e APP_ENV=testing php artisan migrate:fresh --yes +docker compose run --rm -e APP_ENV=testing php artisan migrate:fresh --yes ``` --- @@ -321,7 +299,7 @@ bin/phpunit.sh or if using Docker: ``` -docker-compose run --rm php test phpunit +docker compose run --rm php test phpunit ``` Regular PHPUnit arguments are accepted, e.g.: @@ -341,7 +319,8 @@ bin/run_dusk.sh or if using Docker: ``` -docker-compose run --rm php test browser +# `compose exec` doesn't work here due to port conflict with dev instance +docker compose run --rm php test browser ``` --- @@ -364,7 +343,7 @@ yarn karma start --single-run or if using Docker: ``` -docker-compose run --rm php test js +docker compose run --rm php test js ``` # Documentation diff --git a/app/Console/Commands/Ip2AsnUpdate.php b/app/Console/Commands/Ip2AsnUpdate.php new file mode 100644 index 00000000000..9307d97024f --- /dev/null +++ b/app/Console/Commands/Ip2AsnUpdate.php @@ -0,0 +1,24 @@ +. Licensed under the GNU Affero General Public License v3.0. +// See the LICENCE file in the repository root for full licence text. + +namespace App\Console\Commands; + +use App\Libraries\Ip2AsnUpdater; +use Illuminate\Console\Command; + +class Ip2AsnUpdate extends Command +{ + protected $signature = 'ip2asn:update'; + protected $description = 'Update or initialise ip2asn database'; + + public function handle() + { + $this->info('Updating ip2asn database'); + (new Ip2AsnUpdater())->run(function (string $message): void { + $this->info($message); + }); + $this->info('Done'); + } +} diff --git a/app/Console/Commands/NotificationsSendMail.php b/app/Console/Commands/NotificationsSendMail.php index 3a34e90851a..b1ef6dab8a2 100644 --- a/app/Console/Commands/NotificationsSendMail.php +++ b/app/Console/Commands/NotificationsSendMail.php @@ -61,7 +61,14 @@ public function handle() foreach ($userIds->chunk($chunkSize) as $chunk) { $users = User::whereIn('user_id', $chunk)->get(); foreach ($users as $user) { - dispatch(new UserNotificationDigest($user, $fromId, $toId)); + $job = new UserNotificationDigest($user, $fromId, $toId); + try { + $job->handle(); + } catch (\Exception $e) { + // catch exception and queue job to be rerun to avoid job exploding and preventing other notifications from being processed. + log_error($e); + dispatch($job); + } } } diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index ed4528520a4..a3895057ce5 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -25,6 +25,8 @@ class Kernel extends ConsoleKernel Commands\EsIndexScoresSetSchema::class, Commands\EsIndexWiki::class, + Commands\Ip2AsnUpdate::class, + // modding stuff Commands\ModdingRankCommand::class, @@ -88,42 +90,54 @@ class Kernel extends ConsoleKernel protected function schedule(Schedule $schedule) { $schedule->command('store:cleanup-stale-orders') - ->daily(); + ->daily() + ->onOneServer(); $schedule->command('store:expire-products') - ->hourly(); + ->hourly() + ->onOneServer(); $schedule->command('builds:update-propagation-history') - ->everyThirtyMinutes(); + ->everyThirtyMinutes() + ->onOneServer(); $schedule->command('forum:topic-cover-cleanup --yes') ->daily() - ->withoutOverlapping(); + ->withoutOverlapping() + ->onOneServer(); $schedule->command('rankings:recalculate-country-stats') - ->cron('25 0,3,6,9,12,15,18,21 * * *'); + ->cron('25 0,3,6,9,12,15,18,21 * * *') + ->onOneServer(); $schedule->command('modding:rank') - ->cron('*/20 * * * *'); + ->cron('*/20 * * * *') + ->withoutOverlapping() + ->onOneServer(); $schedule->command('oauth:delete-expired-tokens') - ->cron('14 1 * * *'); + ->cron('14 1 * * *') + ->onOneServer(); $schedule->command('notifications:send-mail') ->hourly() - ->withoutOverlapping(); + ->withoutOverlapping() + ->onOneServer(); $schedule->command('user-notifications:cleanup') ->everyThirtyMinutes() - ->withoutOverlapping(); + ->withoutOverlapping() + ->onOneServer(); $schedule->command('notifications:cleanup') ->cron('15,45 * * * *') - ->withoutOverlapping(); + ->withoutOverlapping() + ->onOneServer(); $schedule->command('chat:expire-ack') ->everyFiveMinutes() - ->withoutOverlapping(); + ->withoutOverlapping() + ->onOneServer(); } protected function commands() diff --git a/app/Events/Fulfillments/SupporterTagEvent.php b/app/Events/Fulfillments/SupporterTagEvent.php index 87c22662ce7..bd26ee670b9 100644 --- a/app/Events/Fulfillments/SupporterTagEvent.php +++ b/app/Events/Fulfillments/SupporterTagEvent.php @@ -7,7 +7,8 @@ use App\Events\MessageableEvent; use App\Models\Store\Order; -use ArrayAccess; +use App\Models\Store\OrderItem; +use App\Models\Store\Product; use Sentry\State\Scope; class SupporterTagEvent implements HasOrder, MessageableEvent @@ -15,10 +16,10 @@ class SupporterTagEvent implements HasOrder, MessageableEvent /** @var Order */ protected $order; - /** @var ArrayAccess */ + /** @var iterable */ private $orderItems; - public function __construct(Order $order, ArrayAccess $orderItems) + public function __construct(Order $order, iterable $orderItems) { $this->order = $order; $this->orderItems = $orderItems; @@ -34,7 +35,7 @@ public function toMessage() $message = ''; foreach ($this->orderItems as $item) { - if ($item->product->custom_class !== 'supporter-tag') { + if ($item->product->custom_class !== Product::SUPPORTER_TAG_NAME) { // sanity; it shouldn't happen but also make sure it doesn't die. app('sentry')->getClient()->captureMessage( 'SupporterTagEvent order contains non supporter-tag items.', @@ -45,10 +46,11 @@ public function toMessage() continue; } - $duration = (int) $item->extra_data['duration']; - $userId = $item->extra_data['target_id']; + $extraData = $item->extra_data; + $duration = $extraData->duration; + $userId = $extraData->targetId; $userLink = route('users.show', $userId); - $username = $item->extra_data['username']; + $username = $extraData->username; $message .= "\n<{$userLink}|{$username}> ({$userId}) for {$duration} months!"; } diff --git a/app/Events/NewPrivateNotificationEvent.php b/app/Events/NewPrivateNotificationEvent.php index 39a4321f996..5374622220a 100644 --- a/app/Events/NewPrivateNotificationEvent.php +++ b/app/Events/NewPrivateNotificationEvent.php @@ -13,17 +13,14 @@ class NewPrivateNotificationEvent extends NotificationEventBase { use SerializesModels; - public $notification; - private $receiverIds; - /** * Create a new event instance. * * @return void */ - public function __construct(Notification $notification, array $receiverIds) + public function __construct(public Notification $notification, private array $receiverIds) { - parent::__construct($notification); + parent::__construct(); $this->notification = $notification; $this->receiverIds = $receiverIds; diff --git a/app/Exceptions/ModelNotSavedException.php b/app/Exceptions/ModelNotSavedException.php index 5f56dd4a160..81db87830d2 100644 --- a/app/Exceptions/ModelNotSavedException.php +++ b/app/Exceptions/ModelNotSavedException.php @@ -5,9 +5,26 @@ namespace App\Exceptions; +use Exception; +use Illuminate\Http\Response; + // This is used for model's saveOrExplode class ModelNotSavedException extends SilencedException { + public static function makeResponse(?Exception $e, array $modelFields): Response + { + $json = [ + 'error' => $e?->getMessage(), + 'form_error' => [], + ]; + + foreach ($modelFields as $field => $model) { + $json['form_error'][$field] = $model->validationErrors()->all(); + } + + return response($json, 422); + } + public function getStatusCode() { return 422; diff --git a/app/Http/Controllers/AccountController.php b/app/Http/Controllers/AccountController.php index 27635db31e0..f0df6ad0b22 100644 --- a/app/Http/Controllers/AccountController.php +++ b/app/Http/Controllers/AccountController.php @@ -15,6 +15,8 @@ use App\Models\UserAccountHistory; use App\Models\UserNotificationOption; use App\Transformers\CurrentUserTransformer; +use App\Transformers\LegacyApiKeyTransformer; +use App\Transformers\LegacyIrcKeyTransformer; use Auth; use DB; use Mail; @@ -111,12 +113,20 @@ public function edit() $authorizedClients = json_collection(Client::forUser($user), 'OAuth\Client', 'user'); $ownClients = json_collection($user->oauthClients()->where('revoked', false)->get(), 'OAuth\Client', ['redirect', 'secret']); + $legacyApiKey = $user->apiKeys()->available()->first(); + $legacyApiKeyJson = $legacyApiKey === null ? null : json_item($legacyApiKey, new LegacyApiKeyTransformer()); + + $legacyIrcKey = $user->legacyIrcKey; + $legacyIrcKeyJson = $legacyIrcKey === null ? null : json_item($legacyIrcKey, new LegacyIrcKeyTransformer()); + $notificationOptions = $user->notificationOptions->keyBy('name'); return ext_view('accounts.edit', compact( 'authorizedClients', 'blocks', 'currentSessionId', + 'legacyApiKeyJson', + 'legacyIrcKeyJson', 'notificationOptions', 'ownClients', 'sessions' @@ -140,7 +150,7 @@ public function update() try { $user->fill($params)->saveOrExplode(); } catch (ModelNotSavedException $e) { - return $this->errorResponse($user, $e); + return ModelNotSavedException::makeResponse($e, compact('user')); } return json_item($user, new CurrentUserTransformer()); @@ -165,7 +175,7 @@ public function updateEmail() return response([], 204); } else { - return $this->errorResponse($user); + return ModelNotSavedException::makeResponse(null, compact('user')); } } @@ -240,7 +250,10 @@ public function updateOptions() $user->profileCustomization()->fill($profileParams)->saveOrExplode(); } } catch (ModelNotSavedException $e) { - return $this->errorResponse($user, $e); + return ModelNotSavedException::makeResponse($e, [ + 'user' => $user, + 'user_profile_customization' => $user->profileCustomization(), + ]); } return json_item($user, new CurrentUserTransformer()); @@ -262,7 +275,7 @@ public function updatePassword() return response([], 204); } else { - return $this->errorResponse($user); + return ModelNotSavedException::makeResponse(null, compact('user')); } } @@ -291,12 +304,4 @@ public function reissueCode() { return UserVerification::fromCurrentRequest()->reissue(); } - - private function errorResponse($user, $exception = null) - { - return response([ - 'form_error' => ['user' => $user->validationErrors()->all()], - 'error' => optional($exception)->getMessage(), - ], 422); - } } diff --git a/app/Http/Controllers/ArtistTracksController.php b/app/Http/Controllers/ArtistTracksController.php index 3cd5764f8fa..4c2fc80d8bd 100644 --- a/app/Http/Controllers/ArtistTracksController.php +++ b/app/Http/Controllers/ArtistTracksController.php @@ -8,6 +8,7 @@ use App\Libraries\Search\ArtistTrackSearch; use App\Libraries\Search\ArtistTrackSearchParamsFromRequest; use App\Models\ArtistTrack; +use App\Transformers\ArtistTrackTransformer; class ArtistTracksController extends Controller { @@ -16,14 +17,15 @@ public function index() $params = ArtistTrackSearchParamsFromRequest::fromArray(request()->all()); $search = new ArtistTrackSearch($params); - $data = [ - 'artist_tracks' => json_collection($search->records(), 'ArtistTrack', ['artist', 'album']), + $tracks = $search->records(); + $index = [ + 'artist_tracks' => json_collection($tracks, new ArtistTrackTransformer(), ['artist', 'album']), 'search' => ArtistTrackSearchParamsFromRequest::toArray($params), - 'cursor' => $search->getSortCursor(), + ...cursor_for_response($search->getSortCursor()), ]; if (is_json_request()) { - return $data; + return $index; } $availableGenres = cache_remember_mutexed( @@ -33,7 +35,7 @@ public function index() fn () => ArtistTrack::distinct()->pluck('genre')->sort()->values(), ); - return ext_view('artist_tracks.index', compact('availableGenres', 'data')); + return ext_view('artist_tracks.index', compact('availableGenres', 'index')); } public function show($id) diff --git a/app/Http/Controllers/BeatmapDiscussionsController.php b/app/Http/Controllers/BeatmapDiscussionsController.php index 40cef2707e0..71adb61614c 100644 --- a/app/Http/Controllers/BeatmapDiscussionsController.php +++ b/app/Http/Controllers/BeatmapDiscussionsController.php @@ -20,7 +20,7 @@ class BeatmapDiscussionsController extends Controller { public function __construct() { - $this->middleware('auth', ['except' => ['index', 'show']]); + $this->middleware('auth', ['except' => ['index', 'mediaUrl', 'show']]); $this->middleware('require-scopes:public', ['only' => ['index']]); parent::__construct(); @@ -117,6 +117,14 @@ public function index() return ext_view('beatmap_discussions.index', compact('json', 'search', 'paginator')); } + public function mediaUrl() + { + $url = get_string(request('url')); + + // Tell browser not to request url for a while. + return redirect(proxy_media($url))->header('Cache-Control', 'max-age=600'); + } + public function restore($id) { $discussion = BeatmapDiscussion::whereNotNull('deleted_at')->findOrFail($id); diff --git a/app/Http/Controllers/BeatmapPacksController.php b/app/Http/Controllers/BeatmapPacksController.php index 5338172a5d1..38017168ac7 100644 --- a/app/Http/Controllers/BeatmapPacksController.php +++ b/app/Http/Controllers/BeatmapPacksController.php @@ -12,7 +12,7 @@ class BeatmapPacksController extends Controller { - private const PER_PAGE = 20; + private const PER_PAGE = 100; public function index() { @@ -32,30 +32,19 @@ public function show($idOrTag) { $query = BeatmapPack::default(); - if (!ctype_digit($idOrTag)) { - $pack = $query->where('tag', $idOrTag)->firstOrFail(); + if (ctype_digit($idOrTag)) { + $pack = $query->findOrFail($idOrTag); return ujs_redirect(route('packs.show', $pack)); } - $pack = $query->findOrFail($idOrTag); - - return ext_view('packs.show', $this->packData($pack)); - } - - public function raw($id) - { - $pack = BeatmapPack::default()->findOrFail($id); - - return ext_view('packs.raw', $this->packData($pack)); - } - - private function packData($pack) - { + $pack = $query->where('tag', $idOrTag)->firstOrFail(); $mode = Beatmap::modeStr($pack->playmode ?? 0); - $sets = $pack->beatmapsets()->select()->get(); + $sets = $pack->beatmapsets; $userCompletionData = $pack->userCompletionData(Auth::user()); - return compact('mode', 'pack', 'sets', 'userCompletionData'); + $view = request('format') === 'raw' ? 'packs.raw' : 'packs.show'; + + return ext_view($view, compact('mode', 'pack', 'sets', 'userCompletionData')); } } diff --git a/app/Http/Controllers/BeatmapsController.php b/app/Http/Controllers/BeatmapsController.php index 4d064cd83fc..aa412412e74 100644 --- a/app/Http/Controllers/BeatmapsController.php +++ b/app/Http/Controllers/BeatmapsController.php @@ -12,6 +12,7 @@ use App\Models\Beatmap; use App\Models\BeatmapsetEvent; use App\Models\Score\Best\Model as BestModel; +use App\Models\User; use App\Transformers\BeatmapTransformer; use App\Transformers\ScoreTransformer; @@ -270,21 +271,18 @@ public function scores($id) } $params = get_params(request()->all(), null, [ + 'limit:int', 'mode:string', 'mods:string[]', 'type:string', - ]); + ], ['null_missing' => true]); - $mode = presence($params['mode'] ?? null, $beatmap->mode); + $mode = presence($params['mode']) ?? $beatmap->mode; $mods = array_values(array_filter($params['mods'] ?? [])); - $type = presence($params['type'] ?? null, 'global'); + $type = presence($params['type']) ?? 'global'; $currentUser = auth()->user(); - if ($type !== 'global' || !empty($mods)) { - if ($currentUser === null || !$currentUser->isSupporter()) { - throw new InvariantException(osu_trans('errors.supporter_only')); - } - } + $this->assertSupporterOnlyOptions($currentUser, $type, $mods); $query = static::baseScoreQuery($beatmap, $mode, $mods, $type); @@ -297,7 +295,7 @@ public function scores($id) $results = [ 'scores' => json_collection( - $query->visibleUsers()->forListing(), + $query->visibleUsers()->forListing($params['limit']), $scoreTransformer, static::DEFAULT_SCORE_INCLUDES ), @@ -342,6 +340,7 @@ public function soloScores($id) } $params = get_params(request()->all(), null, [ + 'limit:int', 'mode', 'mods:string[]', 'type:string', @@ -358,15 +357,12 @@ public function soloScores($id) $type = presence($params['type'], 'global'); $currentUser = auth()->user(); - if ($type !== 'global' || !empty($mods)) { - if ($currentUser === null || !$currentUser->isSupporter()) { - throw new InvariantException(osu_trans('errors.supporter_only')); - } - } + $this->assertSupporterOnlyOptions($currentUser, $type, $mods); $esFetch = new BeatmapScores([ 'beatmap_ids' => [$beatmap->getKey()], 'is_legacy' => false, + 'limit' => $params['limit'], 'mods' => $mods, 'ruleset_id' => $rulesetId, 'type' => $type, @@ -517,4 +513,15 @@ private static function baseScoreQuery(Beatmap $beatmap, $mode, $mods, $type = n return $query; } + + private function assertSupporterOnlyOptions(?User $currentUser, string $type, array $mods): void + { + $isSupporter = $currentUser?->isSupporter() ?? false; + if ($type !== 'global' && !$isSupporter) { + throw new InvariantException(osu_trans('errors.supporter_only')); + } + if (!empty($mods) && !is_api_request() && !$isSupporter) { + throw new InvariantException(osu_trans('errors.supporter_only')); + } + } } diff --git a/app/Http/Controllers/BeatmapsetsController.php b/app/Http/Controllers/BeatmapsetsController.php index 978c156f17e..a9b5d5d603b 100644 --- a/app/Http/Controllers/BeatmapsetsController.php +++ b/app/Http/Controllers/BeatmapsetsController.php @@ -72,7 +72,11 @@ public function lookup() public function show($id) { - $beatmapset = Beatmapset::whereHas('beatmaps')->findOrFail($id); + $beatmapset = ( + priv_check('BeatmapsetShowDeleted')->can() + ? Beatmapset::withTrashed()->whereHas('allBeatmaps') + : Beatmapset::whereHas('beatmaps') + )->findOrFail($id); $set = $this->showJson($beatmapset); @@ -275,26 +279,33 @@ public function update($id) 'nsfw:bool', ]); - $offsetParams = get_params($params, 'beatmapset', [ - 'offset:int', - ]); - - $updateParams = array_merge($metadataParams, $offsetParams); - if (count($metadataParams) > 0) { priv_check('BeatmapsetMetadataEdit', $beatmapset)->ensureCan(); } - if (count($offsetParams) > 0) { + $updateParams = [ + ...$metadataParams, + ...get_params($params, 'beatmapset', [ + 'offset:int', + 'tags:string', + ]), + ]; + + if (array_key_exists('offset', $updateParams)) { priv_check('BeatmapsetOffsetEdit')->ensureCan(); } + if (array_key_exists('tags', $updateParams)) { + priv_check('BeatmapsetTagsEdit')->ensureCan(); + } + if (count($updateParams) > 0) { DB::transaction(function () use ($beatmapset, $updateParams) { $oldGenreId = $beatmapset->genre_id; $oldLanguageId = $beatmapset->language_id; $oldNsfw = $beatmapset->nsfw; $oldOffset = $beatmapset->offset; + $oldTags = $beatmapset->tags; $user = auth()->user(); $beatmapset->fill($updateParams)->saveOrExplode(); @@ -326,6 +337,13 @@ public function update($id) 'new' => $beatmapset->offset, ])->saveOrExplode(); } + + if ($oldTags !== $beatmapset->tags) { + BeatmapsetEvent::log(BeatmapsetEvent::TAGS_EDIT, $user, $beatmapset, [ + 'old' => $oldTags, + 'new' => $beatmapset->tags, + ])->saveOrExplode(); + } }); } @@ -348,7 +366,7 @@ private function getSearchResponse(?array $params = null) 'beatmapsets' => json_collection( $records, new BeatmapsetTransformer(), - 'beatmaps.max_combo' + ['beatmaps.max_combo', 'pack_tags'] ), 'search' => [ 'sort' => $search->getParams()->getSort(), @@ -363,10 +381,13 @@ private function getSearchResponse(?array $params = null) private function showJson($beatmapset) { + $beatmapRelation = $beatmapset->trashed() + ? 'allBeatmaps' + : 'beatmaps'; $beatmapset->load([ - 'beatmaps.baseDifficultyRatings', - 'beatmaps.baseMaxCombo', - 'beatmaps.failtimes', + "{$beatmapRelation}.baseDifficultyRatings", + "{$beatmapRelation}.baseMaxCombo", + "{$beatmapRelation}.failtimes", 'genre', 'language', 'user', @@ -386,6 +407,7 @@ private function showJson($beatmapset) 'description', 'genre', 'language', + 'pack_tags', 'ratings', 'recent_favourites', 'related_users', diff --git a/app/Http/Controllers/ChangelogController.php b/app/Http/Controllers/ChangelogController.php index 9d4446a77cb..51580b3aba3 100644 --- a/app/Http/Controllers/ChangelogController.php +++ b/app/Http/Controllers/ChangelogController.php @@ -204,7 +204,7 @@ public function github() { $token = config('osu.changelog.github_token'); - $signatureHeader = explode('=', request()->header('X-Hub-Signature')); + $signatureHeader = explode('=', request()->header('X-Hub-Signature') ?? ''); if (count($signatureHeader) !== 2) { abort(422, 'invalid signature header'); diff --git a/app/Http/Controllers/Chat/Channels/MessagesController.php b/app/Http/Controllers/Chat/Channels/MessagesController.php index 54106dd2abc..5a7d70622a0 100644 --- a/app/Http/Controllers/Chat/Channels/MessagesController.php +++ b/app/Http/Controllers/Chat/Channels/MessagesController.php @@ -129,8 +129,9 @@ public function index($channelId) return [ 'messages' => json_collection($messages, new MessageTransformer()), + // FIXME: messages with null used should be removed from db... 'users' => json_collection( - $messages->pluck('sender')->uniqueStrict('user_id')->values(), + $messages->pluck('sender')->filter()->uniqueStrict('user_id')->values(), new UserCompactTransformer() ), ]; diff --git a/app/Http/Controllers/Chat/ChannelsController.php b/app/Http/Controllers/Chat/ChannelsController.php index 56199e83f73..1d0beb806a2 100644 --- a/app/Http/Controllers/Chat/ChannelsController.php +++ b/app/Http/Controllers/Chat/ChannelsController.php @@ -84,17 +84,17 @@ public function index() public function join($channelId, $userId) { $channel = Channel::where('channel_id', $channelId)->firstOrFail(); - $user = auth()->user(); + $currentUser = auth()->user(); priv_check('ChatChannelJoin', $channel)->ensureCan(); - if ($user->user_id !== get_int($userId)) { + if ($currentUser->getKey() !== get_int($userId)) { abort(403); } - $channel->addUser(Auth::user()); + $channel->addUser($currentUser); - return json_item($channel, ChannelTransformer::forUser($user), ChannelTransformer::LISTING_INCLUDES); + return json_item($channel, ChannelTransformer::forUser($currentUser), ChannelTransformer::LISTING_INCLUDES); } /** @@ -214,31 +214,29 @@ public function show($channelId) /** * Create Channel * - * TODO: description needs fixing. - * - * This endpoint creates a new channel if doesn't exist and joins it. - * Currently only for rejoining existing PM channels which the user has left. + * Creates a new PM or announcement channel. + * Rejoins the PM channel if it already exists. * * --- * * ### Response Format * - * Returns [ChatChannel](#chatchannel) with `recent_messages` attribute. - * Note that in the case of `PM`s, if there's no existing PM channel, most of the fields will be blank. - * In that case, [send a message](#create-new-pm) instead to create the channel. + * Returns [ChatChannel](#chatchannel) with `recent_messages` attribute; `recent_messages` is deprecated and should not be used. * * @bodyParam channel object channel details; required if `type` is `ANNOUNCE`. No-example * @bodyParam channel.name string the channel name; required if `type` is `ANNOUNCE`. No-example * @bodyParam channel.description string the channel description; required if `type` is `ANNOUNCE`. No-example * @bodyParam message string message to send with the announcement; required if `type` is `ANNOUNCE`. No-example * @bodyParam target_id integer target user id; required if `type` is `PM`; ignored, otherwise. Example: 2 - * @bodyParam target_ids integer[] target user ids; required if `type` is `PM`; ignored, otherwise. No-example + * @bodyParam target_ids integer[] target user ids; required if `type` is `ANNOUNCE`; ignored, otherwise. No-example * @bodyParam type string required channel type (currently only supports `PM` and `ANNOUNCE`) Example: PM * * @response { * "channel_id": 1, + * "description": "best channel", + * "icon": "https://a.ppy.sh/2?1519081077.png", + * "moderated": false, * "name": "#pm_1-2", - * "description": "", * "type": "PM", * "recent_messages": [ * { @@ -247,7 +245,18 @@ public function show($channelId) * "channel_id": 1, * "timestamp": "2020-01-01T00:00:00+00:00", * "content": "Happy new year", - * "is_action": false + * "is_action": false, + * "sender": { + * "id": 2, + * "username": "peppy", + * "profile_colour": "#3366FF", + * "avatar_url": "https://a.ppy.sh/2?1519081077.png", + * "country_code": "AU", + * "is_active": true, + * "is_bot": false, + * "is_online": true, + * "is_supporter": true + * } * } * ] * } diff --git a/app/Http/Controllers/Chat/ChatController.php b/app/Http/Controllers/Chat/ChatController.php index e9f4b78c19d..7ed19a59ec3 100644 --- a/app/Http/Controllers/Chat/ChatController.php +++ b/app/Http/Controllers/Chat/ChatController.php @@ -28,6 +28,25 @@ public function __construct() parent::__construct(); } + /** + * Chat Keepalive + * + * Request periodically to reset chat activity timeout. Also returns an updated list of recent silences. + * + * See [Public channels and activity timeout](#public-channels-and-activity-timeout) + * + * --- + * + * ### Response Format + * + * Field | Type + * ---------------- | ----------------- + * silences | [UserSilence](#usersilence)[] + * + * @queryParam history_since integer [UserSilence](#usersilence)s after the specified id to return. + * This field is preferred and takes precedence over `since`. + * @queryParam since integer [UserSilence](#usersilence)s after the specified [ChatMessage.message_id](#chatmessage) to return. No-example + */ public function ack() { Chat::ack(auth()->user()); @@ -53,9 +72,9 @@ public function ack() * * Field | Type * ---------------- | ----------------- - * new_channel_id | `channel_id` of newly created [ChatChannel](#chatchannel) - * presence | array of [ChatChannel](#chatchannel) + * channel | The new [ChatChannel](#chatchannel) * message | the sent [ChatMessage](#chatmessage) + * new_channel_id | Deprecated; `channel_id` of newly created [ChatChannel](#chatchannel) * *