Merge branch 'master' into perf_build

.pre-commit-config.yaml

@@ -57,7 +57,7 @@ repos:
 
       - id: cspell-modified-files
         name: check spelling of files
-        entry: sh -c "npx cspell --config .cspell.json `git diff --cached -p --name-status | cut -c3- | tr '\n' ' '`"
+        entry: sh -c "npx cspell --no-must-find-files --config .cspell.json `git diff --cached -p --name-status | cut -c3- | tr '\n' ' '`"
         language: system
         stages: [pre-commit]
 

dashboard/src2/components/NavigationItems.vue

@@ -135,8 +135,7 @@ export default {
 							name: 'DB Analyzer',
 							icon: () => h(Activity),
 							route: '/database-analyzer',
-							isActive: routeName === 'DB Analyzer',
-							condition: this.$team.doc?.is_desk_user
+							isActive: routeName === 'DB Analyzer'
 						}
 					].filter(item => item.condition ?? true),
 					isActive: ['SQL Playground', 'DB Analyzer', 'Log Browser'].includes(

dashboard/src2/components/server/ServerCharts.vue

@@ -98,7 +98,7 @@
 					:key="spaceData"
 					:data="spaceData"
 					unit="%"
-					:chartTheme="[$theme.colors.red[500]]"
+					:chartTheme="[$theme.colors.red[500], $theme.colors.yellow[400]]"
 					:loading="$resources.space.loading"
 					:error="$resources.space.error"
 					:showCard="false"
@@ -128,20 +128,22 @@
 					:key="iopsData"
 					:data="iopsData"
 					unit="I0ps"
-					:chartTheme="[$theme.colors.purple[500]]"
+					:chartTheme="[$theme.colors.purple[500], $theme.colors.blue[500]]"
 					:loading="$resources.iops.loading"
 					:error="$resources.iops.error"
 					:showCard="false"
 					class="h-[15.55rem] p-2 pb-3"
 				/>
 			</AnalyticsCard>
 		</div>
-		<div class="!mt-6 flex space-x-2">
+		<div
+			class="!mt-6 flex w-fit cursor-pointer space-x-2"
+			@click="toggleAdvancedAnalytics"
+		>
 			<h2 class="text-lg font-semibold">Advanced Analytics</h2>
 			<FeatherIcon
-				class="h-5 w-5 cursor-pointer text-gray-500 hover:text-gray-700"
+				class="h-5 w-5 text-gray-500 hover:text-gray-700"
 				:name="showAdvancedAnalytics ? 'chevron-down' : 'chevron-right'"
-				@click="toggleAdvancedAnalytics"
 			/>
 		</div>
 
@@ -718,13 +720,13 @@ export default {
 			let iops = this.$resources.iops.data;
 			if (!iops) return;
 
-			return this.transformSingleLineChartData(iops);
+			return this.transformMultiLineChartData(iops);
 		},
 		spaceData() {
 			let space = this.$resources.space.data;
 			if (!space) return;
 
-			return this.transformSingleLineChartData(space, true);
+			return this.transformMultiLineChartData(space);
 		},
 		networkData() {
 			let network = this.$resources.network.data;

dashboard/src2/components/site/SiteAnalytics.vue

@@ -74,12 +74,14 @@
 			</AnalyticsCard>
 		</div>
 
-		<div class="!mt-6 flex space-x-2">
+		<div
+			class="!mt-6 flex w-fit cursor-pointer space-x-2"
+			@click="toggleAdvancedAnalytics"
+		>
 			<h2 class="text-lg font-semibold">Advanced Analytics</h2>
 			<FeatherIcon
-				class="h-5 w-5 cursor-pointer text-gray-500 hover:text-gray-700"
+				class="h-5 w-5 text-gray-500 hover:text-gray-700"
 				:name="showAdvancedAnalytics ? 'chevron-down' : 'chevron-right'"
-				@click="toggleAdvancedAnalytics"
 			/>
 		</div>
 

dashboard/src2/components/site/performance/SiteSlowQueries.vue

@@ -1,36 +1,49 @@
 <template>
 	<div>
 		<PerformanceReport
-			v-if="siteVersion !== 'Version 14'"
 			title="Slow Queries"
 			:site="name"
 			:reportOptions="slowQueriesOptions"
 		/>
-		<div v-else class="m-5 space-y-4">
-			<AlertBanner
-				title="Performance Tuning coming soon for Version 15"
-				type="info"
-			/>
-		</div>
+		<SiteSlowQueryDialog
+			v-if="show"
+			v-model="show"
+			:query="selectedQuery"
+			:duration="selectedQueryDuration"
+		/>
 	</div>
 </template>
 
 <script>
 import AlertBanner from '../../AlertBanner.vue';
 import PerformanceReport from './PerformanceReport.vue';
+import SiteSlowQueryDialog from './SiteSlowQueryDialog.vue';
 
 export default {
 	props: ['name', 'siteVersion'],
 	components: {
 		PerformanceReport,
-		AlertBanner
+		AlertBanner,
+		SiteSlowQueryDialog
+	},
+	data() {
+		return {
+			show: false,
+			selectedQuery: '',
+			selectedQueryDuration: 0
+		};
 	},
 	computed: {
 		slowQueriesOptions() {
 			return {
 				experimental: true,
 				documentation: 'https://frappecloud.com/docs/performance-tuning',
 				data: () => this.$resources.slowQueries.data.data,
+				onRowClick: row => {
+					this.selectedQuery = row.query;
+					this.selectedQueryDuration = row.duration;
+					this.show = true;
+				},
 				emptyStateMessage: 'No slow queries found',
 				columns: [
 					{

dashboard/src2/components/site/performance/SiteSlowQueryDialog.vue

@@ -0,0 +1,25 @@
+<template>
+	<Dialog
+		:options="{
+			title: 'Normalized Query',
+			size: '2xl'
+		}"
+		v-model="show"
+	>
+		<template #body-content>
+			<pre
+				class="mt-2 whitespace-pre-wrap rounded-lg border-2 border-gray-200 bg-gray-100 p-3 text-sm text-gray-700"
+				>{{ query }}</pre
+			>
+			<div class="mt-2 flex text-sm">
+				<span class="ml-auto">Duration: {{ duration.toFixed(2) }} seconds</span>
+			</div>
+		</template>
+	</Dialog>
+</template>
+
+<script>
+export default {
+	props: ['query', 'duration']
+};
+</script>

press/api/github.py

@@ -128,7 +128,7 @@ def installations(token):
 				}
 			)
 	else:
-		frappe.throw(data.get("message") or "An error occured")
+		frappe.throw(data.get("message") or "An error Occurred")
 
 	return installations
 
@@ -174,9 +174,7 @@ def repository(owner, name, installation=None):
 	headers = {
 		"Authorization": f"token {token}",
 	}
-	repo = requests.get(
-		f"https://api.github.com/repos/{owner}/{name}", headers=headers
-	).json()
+	repo = requests.get(f"https://api.github.com/repos/{owner}/{name}", headers=headers).json()
 
 	current_page, is_last_page = 1, False
 	branches = []

press/api/server.py

@@ -29,6 +29,9 @@ def poly_get_doc(doctypes, name):
 	return frappe.get_doc(doctypes[-1], name)
 
 
+MOUNTPOINT_REGEX = "(/|/opt/volumes/mariadb|/opt/volumes/benches)"
+
+
 @frappe.whitelist()
 def all(server_filter=None):  # noqa: C901
 	if server_filter is None:
@@ -193,7 +196,7 @@ def usage(name):
 			lambda x: x,
 		),
 		"disk": (
-			f"""(node_filesystem_size_bytes{{instance="{name}", job="node", mountpoint="/"}} - node_filesystem_avail_bytes{{instance="{name}", job="node", mountpoint="/"}}) / (1024 * 1024 * 1024)""",
+			f"""sum(node_filesystem_size_bytes{{instance="{name}", job="node", mountpoint=~"{MOUNTPOINT_REGEX}"}} - node_filesystem_avail_bytes{{instance="{name}", job="node", mountpoint=~"{MOUNTPOINT_REGEX}"}}) by ()/ (1024 * 1024 * 1024)""",
 			lambda x: x,
 		),
 		"memory": (
@@ -218,7 +221,7 @@ def total_resource(name):
 			lambda x: x,
 		),
 		"disk": (
-			f"""(node_filesystem_size_bytes{{instance="{name}", job="node", mountpoint="/"}}) / (1024 * 1024 * 1024)""",
+			f"""sum(node_filesystem_size_bytes{{instance="{name}", job="node", mountpoint=~"{MOUNTPOINT_REGEX}"}}) by () / (1024 * 1024 * 1024)""",
 			lambda x: x,
 		),
 		"memory": (
@@ -289,7 +292,7 @@ def analytics(name, query, timezone, duration):
 			lambda x: x["device"],
 		),
 		"space": (
-			f"""100 - ((node_filesystem_avail_bytes{{instance="{name}", job="node", mountpoint="/"}} * 100) / node_filesystem_size_bytes{{instance="{name}", job="node", mountpoint="/"}})""",
+			f"""100 - ((node_filesystem_avail_bytes{{instance="{name}", job="node", mountpoint=~"{MOUNTPOINT_REGEX}"}} * 100) / node_filesystem_size_bytes{{instance="{name}", job="node", mountpoint=~"{MOUNTPOINT_REGEX}"}})""",
 			lambda x: x["mountpoint"],
 		),
 		"loadavg": (
@@ -549,7 +552,7 @@ def rename(name, title):
 	doc.save()
 
 
-def get_timespan_timegrain(duration: str) -> Tuple[int, int]:
+def get_timespan_timegrain(duration: str) -> tuple[int, int]:
 	timespan, timegrain = {
 		"1 Hour": (60 * 60, 2 * 60),
 		"6 Hour": (6 * 60 * 60, 5 * 60),

press/docker/config/redis-cache.conf

@@ -3,7 +3,7 @@ dir /home/frappe/frappe-bench/config/pids
 pidfile /home/frappe/frappe-bench/config/pids/redis-cache.pid
 bind 127.0.0.1
 port 13000
-maxmemory 512mb
+maxmemory {{ doc.redis_cache_size }}mb
 maxmemory-policy allkeys-lru
 appendonly no
 save ""

press/fixtures/press_job_type.json

@@ -2,7 +2,7 @@
  {
   "docstatus": 0,
   "doctype": "Press Job Type",
-  "modified": "2024-07-30 16:26:28.217044",
+  "modified": "2025-01-02 15:31:03.095292",
   "name": "Create Server",
   "steps": [
    {
@@ -61,7 +61,7 @@
     "wait_until_true": 1
    },
    {
-    "script": "if doc.server_type == \"Database Server\":\n    default_variables = frappe.get_all(\"MariaDB Variable\", {\"set_on_new_servers\":1}, pluck=\"name\")\n    for var_name in default_variables:\n        var = frappe.get_doc(\"MariaDB Variable\", var_name)\n        var.set_on_server(doc.server)\n\nserver = frappe.get_doc(doc.server_type, doc.server)\nserver.set_swappiness()\nserver.add_glass_file()\nserver.update_filebeat()\n\nif doc.server_type == \"Server\":\n    server.setup_mysqldump()\n    server.install_earlyoom()\n",
+    "script": "if doc.server_type == \"Database Server\":\n    default_variables = frappe.get_all(\"MariaDB Variable\", {\"set_on_new_servers\":1}, pluck=\"name\")\n    for var_name in default_variables:\n        var = frappe.get_doc(\"MariaDB Variable\", var_name)\n        var.set_on_server(doc.server)\n\nserver = frappe.get_doc(doc.server_type, doc.server)\nserver.set_swappiness()\nserver.add_glass_file()\nserver.update_filebeat()\n\nif doc.server_type == \"Server\":\n    server.setup_mysqldump()\n    server.install_earlyoom()\n\nserver.validate_mounts()\nserver.save()",
     "step_name": "Set additional config",
     "wait_until_true": 0
    }
@@ -70,7 +70,7 @@
  {
   "docstatus": 0,
   "doctype": "Press Job Type",
-  "modified": "2024-12-06 10:58:10.573186",
+  "modified": "2025-01-02 15:33:27.252601",
   "name": "Resize Server",
   "steps": [
    {
@@ -109,7 +109,7 @@
     "wait_until_true": 0
    },
    {
-    "script": "machine = frappe.get_doc(\"Virtual Machine\", doc.virtual_machine)\n\nserver = frappe.get_doc(doc.server_type, doc.server)\nif server.plan:\n    plan_disk_size = frappe.db.get_value(\"Plan\", server.plan, \"disk\")\n    if plan_disk_size and plan_disk_size > machine.disk_size:\n        try:\n            server.increase_disk_size(increment=plan_disk_size - machine.disk_size)\n        except:\n            pass",
+    "script": "machine = frappe.get_doc(\"Virtual Machine\", doc.virtual_machine)\n\nserver = frappe.get_doc(doc.server_type, doc.server)\nif server.plan:\n    plan_disk_size = frappe.db.get_value(\"Server Plan\", server.plan, \"disk\")\n    if plan_disk_size and plan_disk_size > machine.disk_size:\n        try:\n            server.increase_disk_size(increment=plan_disk_size - machine.disk_size)\n        except:\n            pass",
     "step_name": "Increase Disk Size",
     "wait_until_true": 0
    }
@@ -212,11 +212,11 @@
  {
   "docstatus": 0,
   "doctype": "Press Job Type",
-  "modified": "2024-08-07 10:54:29.752755",
+  "modified": "2024-12-26 15:15:23.765651",
   "name": "Increase Disk Size",
   "steps": [
    {
-    "script": "server = frappe.get_doc(doc.server_type, doc.server)\nserver.calculated_increase_disk_size()\n    ",
+    "script": "server = frappe.get_doc(doc.server_type, doc.server)\nmountpoint = arguments.labels.get(\"mountpoint\")\nserver.calculated_increase_disk_size(mountpoint=mountpoint)",
     "step_name": "Increase Disk Size",
     "wait_until_true": 0
    },

press/infrastructure/doctype/virtual_machine_migration/virtual_machine_migration.py

@@ -384,8 +384,10 @@ def reset_virtual_machine_attributes(self) -> StepStatus:
 
 		# Set new machine image and machine type
 		machine.virtual_machine_image = self.virtual_machine_image
+		machine.machine_image = None
 		machine.machine_type = self.machine_type
-		machine.disk_size = 10  # Default disk size for new machines
+		machine.root_disk_size = 10  # Default root disk size for new machines
+		machine.has_data_volume = True  # VM Migration always adds a data volume
 		machine.save()
 		return StepStatus.Success
 

press/patches.txt

@@ -135,3 +135,6 @@ press.press.doctype.server.patches.set_plan_and_subscription
 press.patches.v0_7_0.move_site_db_access_users_to_site_db_perm_manager
 press.press.doctype.drip_email.patches.set_correct_field_for_html
 press.patches.v0_7_0.set_label_for_site_database_user
+press.press.doctype.press_settings.patches.set_redis_cache_size
+press.press.doctype.virtual_machine.patches.set_root_disk_size
+press.press.doctype.virtual_machine_image.patches.set_root_size

press/playbooks/roles/extend_ec2_volume/tasks/main.yml

@@ -5,16 +5,30 @@
 - name: Show Block Devices
   command: lsblk
 
+- name: Show Partitions
+  command: 'lsblk --noheadings --output PTTYPE {{ device }}'
+  register: lsblk_output
+
+- name: Set Partition Status
+  set_fact:
+    partitioned_disk: '{{ lsblk_output.stdout.strip() != "" }}'
+
 - name: Extend Partition
-  command: growpart /dev/nvme0n1 1
+  command: 'growpart {{ device }} 1'
   retries: 10
   delay: 10
+  when: partitioned_disk
 
 - name: Show Modified Block Devices
   command: lsblk
 
-- name: Extend Filesystem
-  command: resize2fs /dev/nvme0n1p1
+- name: Extend Partitioned Filesystem
+  command: 'resize2fs {{ device }}-part1'
+  when: partitioned_disk
+
+- name: Extend Un-partitioned Filesystem
+  command: 'resize2fs {{ device }}'
+  when: not partitioned_disk
 
 - name: Add Glass file back
   command: fallocate -l 250M /root/glass

press/press/doctype/agent_job/agent_job.py

@@ -341,6 +341,10 @@ def failed_because_of_agent_update(self) -> bool:
 			return True
 		return False
 
+	@property
+	def on_public_server(self):
+		return bool(frappe.db.get_value(self.server_type, self.server, "public"))
+
 
 def job_detail(job):
 	job = frappe.get_doc("Agent Job", job)