diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 3a7c79e4de31..f0e32cf730b4 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -4692,6 +4692,58 @@ export interface paths { patch?: never; trace?: never; }; + "/api/users/{user_id}/credentials": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Lists all credentials the user has provided */ + get: operations["list_user_credentials_api_users__user_id__credentials_get"]; + put?: never; + /** Allows users to provide credentials for a secret/variable */ + post: operations["provide_credential_api_users__user_id__credentials_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/users/{user_id}/credentials/{user_credentials_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + /** Deletes all credentials for a specific service */ + delete: operations["delete_service_credentials_api_users__user_id__credentials__user_credentials_id__delete"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/users/{user_id}/credentials/{user_credentials_id}/{group_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + /** Deletes a specific credential */ + delete: operations["delete_credentials_api_users__user_id__credentials__user_credentials_id___group_id__delete"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/users/{user_id}/custom_builds": { parameters: { query?: never; @@ -7288,6 +7340,18 @@ export interface components { */ url: string; }; + /** CreateSourceCredentialsPayload */ + CreateSourceCredentialsPayload: { + /** Credentials */ + credentials: components["schemas"]["ServiceCredentialPayload"][]; + /** Source Id */ + source_id: string; + /** + * Source Type + * @constant + */ + source_type: "tool"; + }; /** * CreateType * @enum {string} @@ -7383,6 +7447,27 @@ export interface components { */ username: string; }; + /** CredentialGroupResponse */ + CredentialGroupResponse: { + /** + * Id + * @example 0123456789ABCDEF + */ + id: string; + /** Name */ + name: string; + /** Secrets */ + secrets: components["schemas"]["SecretResponse"][]; + /** Variables */ + variables: components["schemas"]["VariableResponse"][]; + }; + /** CredentialPayload */ + CredentialPayload: { + /** Name */ + name: string; + /** Value */ + value: string | null; + }; /** CustomArchivedHistoryView */ CustomArchivedHistoryView: { /** @@ -15687,6 +15772,18 @@ export interface components { } & { [key: string]: unknown; }; + /** SecretResponse */ + SecretResponse: { + /** Already Set */ + already_set: boolean; + /** + * Id + * @example 0123456789ABCDEF + */ + id: string; + /** Name */ + name: string; + }; /** ServerDirElement */ ServerDirElement: { /** Md5 */ @@ -15805,6 +15902,27 @@ export interface components { */ version: string; }; + /** ServiceCredentialPayload */ + ServiceCredentialPayload: { + /** + * Current Group + * @default default + */ + current_group: string | null; + /** Groups */ + groups: components["schemas"]["ServiceGroupPayload"][]; + /** Reference */ + reference: string; + }; + /** ServiceGroupPayload */ + ServiceGroupPayload: { + /** Name */ + name: string; + /** Secrets */ + secrets: components["schemas"]["CredentialPayload"][]; + /** Variables */ + variables: components["schemas"]["CredentialPayload"][]; + }; /** ServiceType */ ServiceType: { /** @@ -17499,6 +17617,36 @@ export interface components { */ username: string; }; + /** UserCredentialsListResponse */ + UserCredentialsListResponse: components["schemas"]["UserCredentialsResponse"][]; + /** UserCredentialsResponse */ + UserCredentialsResponse: { + /** Current Group Name */ + current_group_name: string; + /** Groups */ + groups: { + [key: string]: components["schemas"]["CredentialGroupResponse"]; + }; + /** + * Id + * @example 0123456789ABCDEF + */ + id: string; + /** Reference */ + reference: string; + /** Source Id */ + source_id: string; + /** + * Source Type + * @constant + */ + source_type: "tool"; + /** + * User Id + * @example 0123456789ABCDEF + */ + user_id: string; + }; /** UserDeletionPayload */ UserDeletionPayload: { /** @@ -17767,6 +17915,18 @@ export interface components { */ username?: string | null; }; + /** VariableResponse */ + VariableResponse: { + /** + * Id + * @example 0123456789ABCDEF + */ + id: string; + /** Name */ + name: string; + /** Value */ + value: string | null; + }; /** Visualization */ Visualization: Record; /** VisualizationCreatePayload */ @@ -33715,6 +33875,192 @@ export interface operations { }; }; }; + list_user_credentials_api_users__user_id__credentials_get: { + parameters: { + query?: { + /** @description The type of source to filter by. */ + source_type?: "tool" | null; + /** @description The ID of the source to filter by. */ + source_id?: string | null; + /** @description The name of the group to filter by. */ + group_name?: string | null; + }; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + user_id: string | "current"; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UserCredentialsListResponse"]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; + provide_credential_api_users__user_id__credentials_post: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + user_id: string | "current"; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateSourceCredentialsPayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UserCredentialsListResponse"]; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; + delete_service_credentials_api_users__user_id__credentials__user_credentials_id__delete: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + user_id: string | "current"; + user_credentials_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; + delete_credentials_api_users__user_id__credentials__user_credentials_id___group_id__delete: { + parameters: { + query?: never; + header?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + "run-as"?: string | null; + }; + path: { + user_id: string | "current"; + user_credentials_id: string; + group_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Request Error */ + "4XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + /** @description Server Error */ + "5XX": { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MessageExceptionModel"]; + }; + }; + }; + }; get_custom_builds_api_users__user_id__custom_builds_get: { parameters: { query?: never; diff --git a/client/src/api/users.ts b/client/src/api/users.ts index a6086985d7a9..893bf23efbcf 100644 --- a/client/src/api/users.ts +++ b/client/src/api/users.ts @@ -1,4 +1,4 @@ -import { GalaxyApi } from "@/api"; +import { type components, GalaxyApi } from "@/api"; import { toQuotaUsage } from "@/components/User/DiskUsage/Quota/model"; import { rethrowSimple } from "@/utils/simple-error"; @@ -35,3 +35,54 @@ export async function fetchCurrentUserQuotaSourceUsage(quotaSourceLabel?: string return toQuotaUsage(data); } + +export type CreateSourceCredentialsPayload = components["schemas"]["CreateSourceCredentialsPayload"]; +export type ServiceCredentialPayload = components["schemas"]["ServiceCredentialPayload"]; +export type ServiceGroupPayload = components["schemas"]["ServiceGroupPayload"]; +export type UserCredentials = components["schemas"]["UserCredentialsResponse"]; + +// TODO: Change API to directly return the correct type to avoid this transformation and additional type definitions. +export function transformToSourceCredentials( + toolId: string, + toolCredentialsDefinition: ServiceCredentialsDefinition[] +): SourceCredentialsDefinition { + return { + sourceType: "tool", + sourceId: toolId, + services: new Map(toolCredentialsDefinition.map((service) => [service.reference, service])), + }; +} + +/** + * Represents the definition of credentials for a particular service. + */ +export interface ServiceCredentialsDefinition { + reference: string; + name: string; + optional: boolean; + multiple: boolean; + label?: string; + description?: string; + variables: ServiceVariableDefinition[]; + secrets: ServiceVariableDefinition[]; +} + +/** + * Represents the definition of credentials for a particular source. + * A source can be a tool, a workflow, etc.Base interface for credentials definitions. + * A source may accept multiple services, each with its own credentials. + */ +export interface SourceCredentialsDefinition { + sourceType: string; + sourceId: string; + services: Map; +} + +/** + * Base interface for credential details. It is used to define the structure of variables and secrets. + */ +export interface ServiceVariableDefinition { + name: string; + label?: string; + description?: string; +} diff --git a/client/src/components/Tool/ToolCard.vue b/client/src/components/Tool/ToolCard.vue index f76955803d2b..a09887a58ee3 100644 --- a/client/src/components/Tool/ToolCard.vue +++ b/client/src/components/Tool/ToolCard.vue @@ -14,6 +14,7 @@ import { useUserStore } from "@/stores/userStore"; import ToolSelectPreferredObjectStore from "./ToolSelectPreferredObjectStore"; import ToolTargetPreferredObjectStorePopover from "./ToolTargetPreferredObjectStorePopover"; +import ToolCredentials from "./ToolCredentials.vue"; import ToolHelpForum from "./ToolHelpForum.vue"; import ToolTutorialRecommendations from "./ToolTutorialRecommendations.vue"; import ToolFavoriteButton from "components/Tool/Buttons/ToolFavoriteButton.vue"; @@ -174,6 +175,12 @@ const showHelpForum = computed(() => isConfigLoaded.value && config.value.enable + +
diff --git a/client/src/components/Tool/ToolCredentials.vue b/client/src/components/Tool/ToolCredentials.vue new file mode 100644 index 000000000000..b19cb56ed689 --- /dev/null +++ b/client/src/components/Tool/ToolCredentials.vue @@ -0,0 +1,209 @@ + + + + + diff --git a/client/src/components/User/Credentials/ManageToolCredentials.vue b/client/src/components/User/Credentials/ManageToolCredentials.vue new file mode 100644 index 000000000000..09e5e6c883ed --- /dev/null +++ b/client/src/components/User/Credentials/ManageToolCredentials.vue @@ -0,0 +1,150 @@ + + + + + diff --git a/client/src/components/User/Credentials/ServiceCredentials.vue b/client/src/components/User/Credentials/ServiceCredentials.vue new file mode 100644 index 000000000000..e9d558a8d05e --- /dev/null +++ b/client/src/components/User/Credentials/ServiceCredentials.vue @@ -0,0 +1,156 @@ + + + + + diff --git a/client/src/stores/userCredentials.ts b/client/src/stores/userCredentials.ts new file mode 100644 index 000000000000..e0b72e8792cb --- /dev/null +++ b/client/src/stores/userCredentials.ts @@ -0,0 +1,77 @@ +import { ref, set } from "vue"; + +import { GalaxyApi } from "@/api"; +import type { CreateSourceCredentialsPayload, UserCredentials } from "@/api/users"; + +import { defineScopedStore } from "./scopedStore"; + +export const useUserCredentialsStore = defineScopedStore("userCredentialsStore", (currentUserId: string) => { + const userCredentialsForTools = ref>({}); + + function getKey(toolId: string): string { + const userId = ensureUserIsRegistered(); + return `${userId}-${toolId}`; + } + + function getAllUserCredentialsForTool(toolId: string): UserCredentials[] | undefined { + ensureUserIsRegistered(); + return userCredentialsForTools.value[toolId]; + } + + async function fetchAllUserCredentialsForTool(toolId: string): Promise { + const userId = ensureUserIsRegistered(); + + const { data, error } = await GalaxyApi().GET("/api/users/{user_id}/credentials", { + params: { + path: { user_id: userId }, + query: { + source_type: "tool", + source_id: toolId, + }, + }, + }); + + if (error) { + throw Error(`Failed to fetch user credentials for tool ${toolId}: ${error.err_msg}`); + } + + const key = getKey(toolId); + set(userCredentialsForTools.value, key, data); + return data; + } + + async function saveUserCredentialsForTool( + providedCredentials: CreateSourceCredentialsPayload + ): Promise { + const userId = ensureUserIsRegistered(); + const toolId = providedCredentials.source_id; + + const { data, error } = await GalaxyApi().POST("/api/users/{user_id}/credentials", { + params: { + path: { user_id: userId }, + }, + body: providedCredentials, + }); + + if (error) { + throw Error(`Failed to save user credentials for tool ${toolId}: ${error.err_msg}`); + } + + const key = getKey(toolId); + set(userCredentialsForTools.value, key, data); + return data; + } + + function ensureUserIsRegistered(): string { + if (currentUserId === "anonymous") { + throw new Error("Only registered users can have tool credentials"); + } + return currentUserId; + } + + return { + getAllUserCredentialsForTool, + fetchAllUserCredentialsForTool, + saveUserCredentialsForTool, + }; +}); diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 2b443cc985a7..e20bdf4081e1 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -11606,6 +11606,73 @@ def __repr__(self): ) +class UserCredentials(Base): + """ + Represents a credential associated with a user for a specific service. + """ + + __tablename__ = "user_credentials" + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("galaxy_user.id"), index=True, nullable=False) + reference: Mapped[str] = mapped_column(nullable=False) + source_type: Mapped[str] = mapped_column(nullable=False) + source_id: Mapped[str] = mapped_column(nullable=False) + current_group_id: Mapped[int] = mapped_column( + ForeignKey("user_credentials_group.id", ondelete="CASCADE"), index=True, nullable=True + ) + create_time: Mapped[Optional[datetime]] = mapped_column(default=now) + update_time: Mapped[Optional[datetime]] = mapped_column(default=now, onupdate=now) + + +class CredentialsGroup(Base): + """ + Represents a group of credentials associated with a user for a specific service. + """ + + __tablename__ = "user_credentials_group" + + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(nullable=False) + user_credentials_id: Mapped[int] = mapped_column(ForeignKey("user_credentials.id", ondelete="CASCADE"), index=True) + create_time: Mapped[Optional[datetime]] = mapped_column(default=now) + update_time: Mapped[Optional[datetime]] = mapped_column(default=now, onupdate=now) + + +class Variable(Base): + """ + Represents a variable associated with a user for a specific service. + """ + + __tablename__ = "credential_variable" + + id: Mapped[int] = mapped_column(primary_key=True) + user_credential_group_id: Mapped[int] = mapped_column( + ForeignKey("user_credentials_group.id", ondelete="CASCADE"), index=True, nullable=False + ) + name: Mapped[str] = mapped_column(nullable=False) + value: Mapped[str] = mapped_column(nullable=False) + create_time: Mapped[Optional[datetime]] = mapped_column(default=now) + update_time: Mapped[Optional[datetime]] = mapped_column(default=now, onupdate=now) + + +class Secret(Base): + """ + Represents a secret associated with a user for a specific service. + """ + + __tablename__ = "credential_secret" + + id: Mapped[int] = mapped_column(primary_key=True) + user_credential_group_id: Mapped[int] = mapped_column( + ForeignKey("user_credentials_group.id", ondelete="CASCADE"), index=True, nullable=False + ) + name: Mapped[str] = mapped_column(nullable=False) + already_set: Mapped[bool] = mapped_column(nullable=False, default=False) + create_time: Mapped[Optional[datetime]] = mapped_column(default=now) + update_time: Mapped[Optional[datetime]] = mapped_column(default=now, onupdate=now) + + # The following models (HDA, LDDA) are mapped imperatively (for details see discussion in PR #12064) # TLDR: there are issues ('metadata' property, Galaxy object wrapping) that need to be addressed separately # before these models can be mapped declaratively. Keeping them in the mapping module breaks the auth package diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/64ee692df15b_add_user_credentials_table.py b/lib/galaxy/model/migrations/alembic/versions_gxy/64ee692df15b_add_user_credentials_table.py new file mode 100644 index 000000000000..4ef732355387 --- /dev/null +++ b/lib/galaxy/model/migrations/alembic/versions_gxy/64ee692df15b_add_user_credentials_table.py @@ -0,0 +1,92 @@ +"""add user credentials table + +Revision ID: 64ee692df15b +Revises: 75348cfb3715 +Create Date: 2024-12-19 10:38:04.970502 + +""" + +from alembic.op import batch_alter_table +from sqlalchemy import ( + Boolean, + Column, + DateTime, + ForeignKey, + Integer, +) + +from galaxy.model.custom_types import TrimmedString +from galaxy.model.database_object_names import build_foreign_key_name +from galaxy.model.migrations.util import ( + create_table, + drop_constraint, + drop_table, +) + +# revision identifiers, used by Alembic. +revision = "64ee692df15b" +down_revision = "75348cfb3715" +branch_labels = None +depends_on = None + + +user_credentials_table = "user_credentials" +user_credentials_group_table = "user_credentials_group" +variable_table = "credential_variable" +secret_table = "credential_secret" + + +def upgrade(): + create_table( + user_credentials_table, + Column("id", Integer, primary_key=True), + Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True, nullable=False), + Column("reference", TrimmedString(255), nullable=False), + Column("source_type", TrimmedString(255), nullable=False), + Column("source_id", TrimmedString(255), nullable=False), + Column("create_time", DateTime), + Column("update_time", DateTime), + ) + create_table( + user_credentials_group_table, + Column("id", Integer, primary_key=True), + Column("name", TrimmedString(255), nullable=False), + Column("user_credentials_id", Integer, ForeignKey("user_credentials.id"), index=True, nullable=False), + Column("create_time", DateTime), + Column("update_time", DateTime), + ) + with batch_alter_table(user_credentials_table) as batch_op: + batch_op.add_column( + Column("current_group_id", Integer, ForeignKey("user_credentials_group.id"), index=True, nullable=True) + ) + create_table( + variable_table, + Column("id", Integer, primary_key=True), + Column( + "user_credential_group_id", Integer, ForeignKey("user_credentials_group.id"), index=True, nullable=False + ), + Column("name", TrimmedString(255), nullable=False), + Column("value", TrimmedString(255), nullable=False), + Column("create_time", DateTime), + Column("update_time", DateTime), + ) + create_table( + secret_table, + Column("id", Integer, primary_key=True), + Column( + "user_credential_group_id", Integer, ForeignKey("user_credentials_group.id"), index=True, nullable=False + ), + Column("name", TrimmedString(255), nullable=False), + Column("already_set", Boolean, nullable=False), + Column("create_time", DateTime), + Column("update_time", DateTime), + ) + + +def downgrade(): + drop_constraint(build_foreign_key_name(user_credentials_table, "current_group_id"), user_credentials_table) + + drop_table(variable_table) + drop_table(secret_table) + drop_table(user_credentials_group_table) + drop_table(user_credentials_table) diff --git a/lib/galaxy/schema/credentials.py b/lib/galaxy/schema/credentials.py new file mode 100644 index 000000000000..c7c62cd7013b --- /dev/null +++ b/lib/galaxy/schema/credentials.py @@ -0,0 +1,70 @@ +from typing import ( + Dict, + List, + Optional, +) + +from pydantic import RootModel +from typing_extensions import Literal + +from galaxy.schema.fields import EncodedDatabaseIdField +from galaxy.schema.schema import Model + +SOURCE_TYPE = Literal["tool"] + + +class CredentialResponse(Model): + id: EncodedDatabaseIdField + name: str + + +class VariableResponse(CredentialResponse): + value: Optional[str] + + +class SecretResponse(CredentialResponse): + already_set: bool + + +class CredentialGroupResponse(Model): + id: EncodedDatabaseIdField + name: str + variables: List[VariableResponse] + secrets: List[SecretResponse] + + +class UserCredentialsResponse(Model): + user_id: EncodedDatabaseIdField + id: EncodedDatabaseIdField + source_type: SOURCE_TYPE + source_id: str + reference: str + current_group_name: str + groups: Dict[str, CredentialGroupResponse] + + +class UserCredentialsListResponse(RootModel): + root: List[UserCredentialsResponse] + + +class CredentialPayload(Model): + name: str + value: Optional[str] + + +class ServiceGroupPayload(Model): + name: str + variables: List[CredentialPayload] + secrets: List[CredentialPayload] + + +class ServiceCredentialPayload(Model): + reference: str # Reference to the service + current_group: Optional[str] = "default" # The selected group, the one that would be used when running the service + groups: List[ServiceGroupPayload] # All provided groups, including the selected one + + +class CreateSourceCredentialsPayload(Model): + source_type: SOURCE_TYPE + source_id: str + credentials: List[ServiceCredentialPayload] # The credentials to create for each service diff --git a/lib/galaxy/tool_util/cwl/parser.py b/lib/galaxy/tool_util/cwl/parser.py index e4a4ff83fb20..3a0a1fd8c805 100644 --- a/lib/galaxy/tool_util/cwl/parser.py +++ b/lib/galaxy/tool_util/cwl/parser.py @@ -78,6 +78,7 @@ "SubworkflowFeatureRequirement", "StepInputExpressionRequirement", "MultipleInputFeatureRequirement", + "CredentialsRequirement", ] @@ -219,6 +220,9 @@ def software_requirements(self) -> List: def resource_requirements(self) -> List: return self.hints_or_requirements_of_class("ResourceRequirement") + def credentials_requirements(self) -> List: + return self.hints_or_requirements_of_class("CredentialsRequirement") + class CommandLineToolProxy(ToolProxy): _class = "CommandLineTool" diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py b/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py index a9dfea4d66d7..54fc6957d45d 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py @@ -29,7 +29,7 @@ def _mulled_build_tool(tool, args): tool_source = get_tool_source(tool) - requirements, *_ = tool_source.parse_requirements_and_containers() + requirements, *_ = tool_source.parse_requirements() targets = requirements_to_mulled_targets(requirements) kwds = args_to_mull_targets_kwds(args) mull_targets(targets, **kwds) diff --git a/lib/galaxy/tool_util/deps/requirements.py b/lib/galaxy/tool_util/deps/requirements.py index bfb02e0ea606..62d32dbfb6ca 100644 --- a/lib/galaxy/tool_util/deps/requirements.py +++ b/lib/galaxy/tool_util/deps/requirements.py @@ -20,6 +20,7 @@ from galaxy.util import ( asbool, + string_as_bool, xml_text, ) from galaxy.util.oset import OrderedSet @@ -305,27 +306,133 @@ def resource_requirements_from_list(requirements: Iterable[Dict[str, Any]]) -> L return rr +class BaseCredential: + def __init__( + self, + name: str, + inject_as_env: str, + label: str = "", + description: str = "", + ) -> None: + self.name = name + self.inject_as_env = inject_as_env + self.label = label + self.description = description + if not self.inject_as_env: + raise ValueError("Missing inject_as_env") + + def to_dict(self) -> Dict[str, Any]: + return { + "name": self.name, + "inject_as_env": self.inject_as_env, + "label": self.label, + "description": self.description, + } + + +class Secret(BaseCredential): + @classmethod + def from_element(cls, elem) -> "Secret": + return cls( + name=elem.get("name"), + inject_as_env=elem.get("inject_as_env"), + label=elem.get("label", ""), + description=elem.get("description", ""), + ) + + +class Variable(BaseCredential): + @classmethod + def from_element(cls, elem) -> "Variable": + return cls( + name=elem.get("name"), + inject_as_env=elem.get("inject_as_env"), + label=elem.get("label", ""), + description=elem.get("description", ""), + ) + + +class CredentialsRequirement: + def __init__( + self, + name: str, + reference: str, + optional: bool = True, + multiple: bool = False, + label: str = "", + description: str = "", + secrets: Optional[List[Secret]] = None, + variables: Optional[List[Variable]] = None, + ) -> None: + self.name = name + self.reference = reference + self.optional = optional + self.multiple = multiple + self.label = label + self.description = description + self.secrets = secrets if secrets is not None else [] + self.variables = variables if variables is not None else [] + + if not self.reference: + raise ValueError("Missing reference") + + def to_dict(self) -> Dict[str, Any]: + return { + "name": self.name, + "reference": self.reference, + "optional": self.optional, + "multiple": self.multiple, + "label": self.label, + "description": self.description, + "secrets": [s.to_dict() for s in self.secrets], + "variables": [v.to_dict() for v in self.variables], + } + + @classmethod + def from_dict(cls, dict: Dict[str, Any]) -> "CredentialsRequirement": + name = dict["name"] + reference = dict["reference"] + optional = dict.get("optional", True) + multiple = dict.get("multiple", False) + label = dict.get("label", "") + description = dict.get("description", "") + secrets = [Secret.from_element(s) for s in dict.get("secrets", [])] + variables = [Variable.from_element(v) for v in dict.get("variables", [])] + return cls( + name=name, + reference=reference, + optional=optional, + multiple=multiple, + label=label, + description=description, + secrets=secrets, + variables=variables, + ) + + def parse_requirements_from_lists( software_requirements: List[Union[ToolRequirement, Dict[str, Any]]], containers: Iterable[Dict[str, Any]], resource_requirements: Iterable[Dict[str, Any]], -) -> Tuple[ToolRequirements, List[ContainerDescription], List[ResourceRequirement]]: + credentials: Iterable[Dict[str, Any]], +) -> Tuple[ToolRequirements, List[ContainerDescription], List[ResourceRequirement], List[CredentialsRequirement]]: return ( ToolRequirements.from_list(software_requirements), [ContainerDescription.from_dict(c) for c in containers], resource_requirements_from_list(resource_requirements), + [CredentialsRequirement.from_dict(s) for s in credentials], ) -def parse_requirements_from_xml(xml_root, parse_resources: bool = False): +def parse_requirements_from_xml(xml_root, parse_resources_and_credentials: bool = False): """ Parses requirements, containers and optionally resource requirements from Xml tree. >>> from galaxy.util import parse_xml_string - >>> def load_requirements(contents, parse_resources=False): + >>> def load_requirements(contents, parse_resources_and_credentials=False): ... contents_document = '''%s''' ... root = parse_xml_string(contents_document % contents) - ... return parse_requirements_from_xml(root, parse_resources=parse_resources) + ... return parse_requirements_from_xml(root, parse_resources_and_credentials=parse_resources_and_credentials) >>> reqs, containers = load_requirements('''bwa''') >>> reqs[0].name 'bwa' @@ -344,8 +451,10 @@ def parse_requirements_from_xml(xml_root, parse_resources: bool = False): requirements_elem = xml_root.find("requirements") requirement_elems = [] + container_elems = [] if requirements_elem is not None: requirement_elems = requirements_elem.findall("requirement") + container_elems = requirements_elem.findall("container") requirements = ToolRequirements() for requirement_elem in requirement_elems: @@ -355,15 +464,13 @@ def parse_requirements_from_xml(xml_root, parse_resources: bool = False): requirement = ToolRequirement(name=name, type=type, version=version) requirements.append(requirement) - container_elems = [] - if requirements_elem is not None: - container_elems = requirements_elem.findall("container") - containers = [container_from_element(c) for c in container_elems] - if parse_resources: + if parse_resources_and_credentials: resource_elems = requirements_elem.findall("resource") if requirements_elem is not None else [] resources = [resource_from_element(r) for r in resource_elems] - return requirements, containers, resources + credentials_elems = requirements_elem.findall("credentials") if requirements_elem is not None else [] + credentials = [credentials_from_element(s) for s in credentials_elems] + return requirements, containers, resources, credentials return requirements, containers @@ -386,3 +493,24 @@ def container_from_element(container_elem) -> ContainerDescription: shell=shell, ) return container + + +def credentials_from_element(credentials_elem) -> CredentialsRequirement: + name = credentials_elem.get("name") + reference = credentials_elem.get("reference") + optional = string_as_bool(credentials_elem.get("optional", "true")) + multiple = string_as_bool(credentials_elem.get("multiple", "false")) + label = credentials_elem.get("label", "") + description = credentials_elem.get("description", "") + secrets = [Secret.from_element(elem) for elem in credentials_elem.findall("secret")] + variables = [Variable.from_element(elem) for elem in credentials_elem.findall("variable")] + return CredentialsRequirement( + name=name, + reference=reference, + optional=optional, + multiple=multiple, + label=label, + description=description, + secrets=secrets, + variables=variables, + ) diff --git a/lib/galaxy/tool_util/linters/cwl.py b/lib/galaxy/tool_util/linters/cwl.py index c72bd433f0aa..6e761dee2fa5 100644 --- a/lib/galaxy/tool_util/linters/cwl.py +++ b/lib/galaxy/tool_util/linters/cwl.py @@ -63,7 +63,7 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): class CWLDockerMissing(Linter): @classmethod def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): - _, containers, *_ = tool_source.parse_requirements_and_containers() + _, containers, *_ = tool_source.parse_requirements() if len(containers) == 0: lint_ctx.warn("Tool does not specify a DockerPull source.") @@ -71,7 +71,7 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): class CWLDockerGood(Linter): @classmethod def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): - _, containers, *_ = tool_source.parse_requirements_and_containers() + _, containers, *_ = tool_source.parse_requirements() if len(containers) > 0: identifier = containers[0].identifier lint_ctx.info(f"Tool will run in Docker image [{identifier}].") diff --git a/lib/galaxy/tool_util/linters/general.py b/lib/galaxy/tool_util/linters/general.py index eb3a98fd3b82..3676dc6a651f 100644 --- a/lib/galaxy/tool_util/linters/general.py +++ b/lib/galaxy/tool_util/linters/general.py @@ -183,7 +183,7 @@ class RequirementNameMissing(Linter): @classmethod def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): _, tool_node = _tool_xml_and_root(tool_source) - requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers() + requirements, *_ = tool_source.parse_requirements() for r in requirements: if r.type != "package": continue @@ -195,7 +195,7 @@ class RequirementVersionMissing(Linter): @classmethod def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): _, tool_node = _tool_xml_and_root(tool_source) - requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers() + requirements, *_ = tool_source.parse_requirements() for r in requirements: if r.type != "package": continue @@ -207,7 +207,7 @@ class RequirementVersionWhitespace(Linter): @classmethod def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): _, tool_node = _tool_xml_and_root(tool_source) - requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers() + requirements, *_ = tool_source.parse_requirements() for r in requirements: if r.type != "package": continue @@ -223,7 +223,7 @@ class ResourceRequirementExpression(Linter): @classmethod def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): _, tool_node = _tool_xml_and_root(tool_source) - requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers() + *_, resource_requirements, _ = tool_source.parse_requirements() for rr in resource_requirements: if rr.runtime_required: lint_ctx.warn( diff --git a/lib/galaxy/tool_util/parser/cwl.py b/lib/galaxy/tool_util/parser/cwl.py index 26d2fdfdc16c..4d9d6d22ae47 100644 --- a/lib/galaxy/tool_util/parser/cwl.py +++ b/lib/galaxy/tool_util/parser/cwl.py @@ -154,7 +154,7 @@ def _parse_output(self, tool, output_instance): output.actions = ToolOutputActionGroup(output, None) return output - def parse_requirements_and_containers(self): + def parse_requirements(self): containers = [] docker_identifier = self.tool_proxy.docker_identifier() if docker_identifier: @@ -162,10 +162,12 @@ def parse_requirements_and_containers(self): software_requirements = self.tool_proxy.software_requirements() resource_requirements = self.tool_proxy.resource_requirements() + credentials = self.tool_proxy.credentials_requirements() return requirements.parse_requirements_from_lists( software_requirements=[{"name": r[0], "version": r[1], "type": "package"} for r in software_requirements], containers=containers, resource_requirements=resource_requirements, + credentials=credentials, ) def parse_profile(self): diff --git a/lib/galaxy/tool_util/parser/interface.py b/lib/galaxy/tool_util/parser/interface.py index af72bf4a4825..10337e9d7b86 100644 --- a/lib/galaxy/tool_util/parser/interface.py +++ b/lib/galaxy/tool_util/parser/interface.py @@ -34,6 +34,7 @@ if TYPE_CHECKING: from galaxy.tool_util.deps.requirements import ( ContainerDescription, + CredentialsRequirement, ResourceRequirement, ToolRequirements, ) @@ -305,10 +306,12 @@ def parse_required_files(self) -> Optional["RequiredFiles"]: return None @abstractmethod - def parse_requirements_and_containers( + def parse_requirements( self, - ) -> Tuple["ToolRequirements", List["ContainerDescription"], List["ResourceRequirement"]]: - """Return triple of ToolRequirement, ContainerDescription and ResourceRequirement lists.""" + ) -> Tuple[ + "ToolRequirements", List["ContainerDescription"], List["ResourceRequirement"], List["CredentialsRequirement"] + ]: + """Return triple of ToolRequirement, ContainerDescription, ResourceRequirement, and CredentialsRequirement objects.""" @abstractmethod def parse_input_pages(self) -> "PagesSource": diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 863a316e45df..7465ff7f1325 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -412,8 +412,8 @@ def parse_include_exclude_list(tag_name): as_dict["excludes"] = parse_include_exclude_list("exclude") return RequiredFiles.from_dict(as_dict) - def parse_requirements_and_containers(self): - return requirements.parse_requirements_from_xml(self.root, parse_resources=True) + def parse_requirements(self): + return requirements.parse_requirements_from_xml(self.root, parse_resources_and_credentials=True) def parse_input_pages(self) -> "XmlPagesSource": return XmlPagesSource(self.root) diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index 88be9c72846a..a64a14e2bc2a 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -109,12 +109,13 @@ def parse_version_command(self): def parse_version_command_interpreter(self): return self.root_dict.get("runtime_version", {}).get("interpreter", None) - def parse_requirements_and_containers(self): + def parse_requirements(self): mixed_requirements = self.root_dict.get("requirements", []) return requirements.parse_requirements_from_lists( software_requirements=[r for r in mixed_requirements if r.get("type") != "resource"], containers=self.root_dict.get("containers", []), resource_requirements=[r for r in mixed_requirements if r.get("type") == "resource"], + credentials=self.root_dict.get("credentials", []), ) def parse_input_pages(self) -> PagesSource: diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 1c23f88066fe..6700dbd18a22 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -600,10 +600,10 @@ practice. @@ -612,6 +612,7 @@ serve as complete descriptions of the runtime of a tool. + @@ -725,6 +726,119 @@ Read more about configuring Galaxy to run Docker jobs + + + + + + + + + +``` +]]> + + + + + + + + The name of the credential set. + + + + + A reference to the source of the credentials. + + + + + The label of the credential set. + + + + + The description of the credential set. + + + + + Whether the credentials are optional for the tool to run. + + + + + Indicates multiple sets of credentials can be provided. + + + + + + + + + + The name of the variable. + + + + + The environment variable name to inject the value as. + + + + + The label for the variable. + + + + + The description for the variable. + + + + + + + + + + The name of the secret. + + + + + The environment variable name to inject the value as. + + + + + The label for the secret. + + + + + The description for the secret. + + + Document type of tool help diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index bb62f87e05d5..995cfe639294 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1216,10 +1216,18 @@ def parse(self, tool_source: ToolSource, guid: Optional[str] = None, dynamic: bo raise Exception(message) # Requirements (dependencies) - requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers() + requirements, containers, resource_requirements, credentials = tool_source.parse_requirements() self.requirements = requirements self.containers = containers self.resource_requirements = resource_requirements + self.credentials = credentials + # for credential in self.credentials: + # pass + # preferences = self.app.config.user_preferences_extra["preferences"] + # main_key, input_key = credential.user_preferences_key.split("/") + # preferences_input = preferences.get(main_key, {}).get("inputs", []) + # if not any(input_item.get("name") == input_key for input_item in preferences_input): + # raise exceptions.ConfigurationError(f"User preferences key {credential.user_preferences_key} not found") required_files = tool_source.parse_required_files() if required_files is None: @@ -2659,6 +2667,27 @@ def to_json(self, trans, kwd=None, job=None, workflow_building_mode=False, histo state_inputs_json: ToolStateDumpedToJsonT = params_to_json(self.inputs, state_inputs, self.app) + credentials = [] + for credential in self.credentials: + credential_dict = credential.to_dict() + credential_dict["variables"] = [ + { + "name": variable.name, + "label": variable.label, + "description": variable.description, + } + for variable in credential.variables + ] + credential_dict["secrets"] = [ + { + "name": secret.name, + "label": secret.label, + "description": secret.description, + } + for secret in credential.secrets + ] + credentials.append(credential_dict) + # update tool model tool_model.update( { @@ -2671,6 +2700,7 @@ def to_json(self, trans, kwd=None, job=None, workflow_building_mode=False, histo "warnings": tool_warnings, "versions": self.tool_versions, "requirements": [{"name": r.name, "version": r.version} for r in self.requirements], + "credentials": credentials, "errors": state_errors, "tool_errors": self.tool_errors, "state_inputs": state_inputs_json, diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 582bd65be06e..e15551ad0771 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -33,6 +33,7 @@ MinimalToolApp, ) from galaxy.tool_util.data import TabularToolDataTable +from galaxy.tool_util.deps.requirements import CredentialsRequirement from galaxy.tools.parameters import ( visit_input_values, wrapped_json, @@ -188,6 +189,21 @@ def set_compute_environment(self, compute_environment: ComputeEnvironment, get_s ) self.execute_tool_hooks(inp_data=inp_data, out_data=out_data, incoming=incoming) + # TODO: provide all information needed (secret value, variable value, current group, etc) to this part... + if hasattr(self.tool, "credentials"): + tool_credentials: List[CredentialsRequirement] = self.tool.credentials + for credentials in tool_credentials: + reference = credentials.reference + current_group = "default" + tool_id = self.tool.id + for secret in credentials.secrets: + vault_ref = f"tool|{tool_id}|{reference}|{current_group}|{secret.name}" + vault_value = f"user_vault.read_secret({vault_ref})" + self.environment_variables.append({"name": secret.inject_as_env, "value": vault_value}) + for variable in credentials.variables: + variable_value = "variable.value" + self.environment_variables.append({"name": variable.inject_as_env, "value": variable_value}) + def execute_tool_hooks(self, inp_data, out_data, incoming): # Certain tools require tasks to be completed prior to job execution # ( this used to be performed in the "exec_before_job" hook, but hooks are deprecated ). diff --git a/lib/galaxy/webapps/galaxy/api/credentials.py b/lib/galaxy/webapps/galaxy/api/credentials.py new file mode 100644 index 000000000000..e5128298c3cd --- /dev/null +++ b/lib/galaxy/webapps/galaxy/api/credentials.py @@ -0,0 +1,98 @@ +""" +API operations on credentials (credentials and variables). +""" + +import logging +from typing import Optional + +from fastapi import ( + Query, + Response, + status, +) + +from galaxy.managers.context import ProvidesUserContext +from galaxy.schema.credentials import ( + CreateSourceCredentialsPayload, + SOURCE_TYPE, + UserCredentialsListResponse, +) +from galaxy.schema.fields import DecodedDatabaseIdField +from galaxy.schema.schema import FlexibleUserIdType +from galaxy.webapps.galaxy.api import ( + depends, + DependsOnTrans, + Router, +) +from galaxy.webapps.galaxy.services.credentials import CredentialsService + +log = logging.getLogger(__name__) + +router = Router(tags=["users"]) + + +@router.cbv +class FastAPICredentials: + service: CredentialsService = depends(CredentialsService) + + @router.get( + "/api/users/{user_id}/credentials", + summary="Lists all credentials the user has provided", + ) + def list_user_credentials( + self, + user_id: FlexibleUserIdType, + trans: ProvidesUserContext = DependsOnTrans, + source_type: Optional[SOURCE_TYPE] = Query( + None, + description="The type of source to filter by.", + ), + source_id: Optional[str] = Query( + None, + description="The ID of the source to filter by.", + ), + group_name: Optional[str] = Query( + None, + description="The name of the group to filter by.", + ), + ) -> UserCredentialsListResponse: + return self.service.list_user_credentials(trans, user_id, source_type, source_id, group_name) + + @router.post( + "/api/users/{user_id}/credentials", + summary="Allows users to provide credentials for a secret/variable", + ) + def provide_credential( + self, + user_id: FlexibleUserIdType, + payload: CreateSourceCredentialsPayload, + trans: ProvidesUserContext = DependsOnTrans, + ) -> UserCredentialsListResponse: + return self.service.provide_credential(trans, user_id, payload) + + @router.delete( + "/api/users/{user_id}/credentials/{user_credentials_id}", + summary="Deletes all credentials for a specific service", + ) + def delete_service_credentials( + self, + user_id: FlexibleUserIdType, + user_credentials_id: DecodedDatabaseIdField, + trans: ProvidesUserContext = DependsOnTrans, + ): + self.service.delete_service_credentials(trans, user_id, user_credentials_id) + return Response(status_code=status.HTTP_204_NO_CONTENT) + + @router.delete( + "/api/users/{user_id}/credentials/{user_credentials_id}/{group_id}", + summary="Deletes a specific credential", + ) + def delete_credentials( + self, + user_id: FlexibleUserIdType, + user_credentials_id: DecodedDatabaseIdField, + group_id: DecodedDatabaseIdField, + trans: ProvidesUserContext = DependsOnTrans, + ): + self.service.delete_credentials(trans, user_id, group_id) + return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/lib/galaxy/webapps/galaxy/services/credentials.py b/lib/galaxy/webapps/galaxy/services/credentials.py new file mode 100644 index 000000000000..0c99559ce0c8 --- /dev/null +++ b/lib/galaxy/webapps/galaxy/services/credentials.py @@ -0,0 +1,298 @@ +from typing import ( + Any, + Dict, + List, + Optional, + Tuple, +) + +from sqlalchemy import select +from sqlalchemy.orm import aliased + +from galaxy import exceptions +from galaxy.managers.context import ProvidesUserContext +from galaxy.model import ( + CredentialsGroup, + Secret, + UserCredentials, + Variable, +) +from galaxy.model.base import transaction +from galaxy.model.scoped_session import galaxy_scoped_session +from galaxy.schema.credentials import ( + CreateSourceCredentialsPayload, + SOURCE_TYPE, + UserCredentialsListResponse, + UserCredentialsResponse, +) +from galaxy.schema.fields import DecodedDatabaseIdField +from galaxy.schema.schema import FlexibleUserIdType +from galaxy.security.vault import UserVaultWrapper +from galaxy.structured_app import StructuredApp + + +class CredentialsService: + """Interface/service object shared by controllers for interacting with credentials.""" + + def __init__(self, app: StructuredApp) -> None: + self._app = app + + def list_user_credentials( + self, + trans: ProvidesUserContext, + user_id: FlexibleUserIdType, + source_type: Optional[SOURCE_TYPE] = None, + source_id: Optional[str] = None, + group_name: Optional[str] = None, + ) -> UserCredentialsListResponse: + """Lists all credentials the user has provided (credentials themselves are not included).""" + db_user_credentials = self._user_credentials( + trans, user_id=user_id, source_type=source_type, source_id=source_id, group_name=group_name + ) + credentials_dict = self._user_credentials_to_dict(db_user_credentials).values() + return UserCredentialsListResponse(root=[UserCredentialsResponse(**cred) for cred in credentials_dict]) + + def provide_credential( + self, + trans: ProvidesUserContext, + user_id: FlexibleUserIdType, + payload: CreateSourceCredentialsPayload, + ) -> UserCredentialsListResponse: + """Allows users to provide credentials for a group of secrets and variables.""" + return self._create_user_credential(trans, user_id, payload) + + def delete_service_credentials( + self, + trans: ProvidesUserContext, + user_id: FlexibleUserIdType, + user_credentials_id: DecodedDatabaseIdField, + ): + """Deletes all credentials for a specific service.""" + user_credentials = self._user_credentials(trans, user_id=user_id, user_credentials_id=user_credentials_id) + rows_to_be_deleted = [item for uc in user_credentials for item in uc] + self._delete_credentials(trans.sa_session, rows_to_be_deleted) + + def delete_credentials( + self, + trans: ProvidesUserContext, + user_id: FlexibleUserIdType, + group_id: DecodedDatabaseIdField, + ): + """Deletes a specific credential group.""" + user_credentials = self._user_credentials(trans, user_id=user_id, group_id=group_id) + rows_to_be_deleted = [item for uc in user_credentials for item in uc if not isinstance(item, UserCredentials)] + self._delete_credentials(trans.sa_session, rows_to_be_deleted) + + def _user_credentials( + self, + trans: ProvidesUserContext, + user_id: FlexibleUserIdType, + source_type: Optional[SOURCE_TYPE] = None, + source_id: Optional[str] = None, + reference: Optional[str] = None, + group_name: Optional[str] = None, + user_credentials_id: Optional[DecodedDatabaseIdField] = None, + group_id: Optional[DecodedDatabaseIdField] = None, + ) -> List[Tuple[UserCredentials, CredentialsGroup, Variable, Secret]]: + if trans.anonymous: + raise exceptions.AuthenticationRequired("You need to be logged in to access your credentials.") + if user_id == "current": + user_id = trans.user.id + if trans.user and trans.user.id != user_id: + raise exceptions.ItemOwnershipException("You can only access your own credentials.") + group_alias = aliased(CredentialsGroup) + var_alias = aliased(Variable) + sec_alias = aliased(Secret) + stmt = ( + select(UserCredentials, group_alias, var_alias, sec_alias) + .join(group_alias, group_alias.user_credentials_id == UserCredentials.id) + .outerjoin(var_alias, var_alias.user_credential_group_id == group_alias.id) + .outerjoin(sec_alias, sec_alias.user_credential_group_id == group_alias.id) + .where(UserCredentials.user_id == user_id) + ) + if source_type: + stmt = stmt.where(UserCredentials.source_type == source_type) + if source_id: + if not source_type: + raise exceptions.RequestParameterInvalidException( + "Source type is required when source ID is provided.", type="error" + ) + stmt = stmt.where(UserCredentials.source_id == source_id) + if group_name: + if not source_type or not source_id: + raise exceptions.RequestParameterInvalidException( + "Source type and source ID are required when group name is provided.", type="error" + ) + stmt = stmt.where(group_alias.name == group_name) + + if reference: + stmt = stmt.where(UserCredentials.reference == reference) + + if user_credentials_id: + stmt = stmt.where(UserCredentials.id == user_credentials_id) + + if group_id: + stmt = stmt.where(group_alias.id == group_id) + + result = trans.sa_session.execute(stmt).all() + return [(row[0], row[1], row[2], row[3]) for row in result] + + def _user_credentials_to_dict( + self, + db_user_credentials: List[Tuple[UserCredentials, CredentialsGroup, Variable, Secret]], + ) -> Dict[int, Dict[str, Any]]: + grouped_data: Dict[int, Dict[str, Any]] = {} + group_name = {group.id: group.name for _, group, _, _ in db_user_credentials} + for user_credentials, credentials_group, variable, secret in db_user_credentials: + grouped_data.setdefault( + user_credentials.id, + dict( + user_id=user_credentials.user_id, + id=user_credentials.id, + reference=user_credentials.reference, + source_type=user_credentials.source_type, + source_id=user_credentials.source_id, + current_group_id=user_credentials.current_group_id, + current_group_name=group_name[user_credentials.current_group_id], + groups={}, + ), + ) + + grouped_data[user_credentials.id]["groups"].setdefault( + credentials_group.name, + dict( + id=credentials_group.id, + name=credentials_group.name, + variables=[], + secrets=[], + ), + ) + + group = grouped_data[user_credentials.id]["groups"][credentials_group.name] + group["secrets"].append({"id": secret.id, "name": secret.name, "already_set": True}) + group["variables"].append({"id": variable.id, "name": variable.name, "value": variable.value}) + + return grouped_data + + def _create_user_credential( + self, + trans: ProvidesUserContext, + user_id: FlexibleUserIdType, + payload: CreateSourceCredentialsPayload, + ) -> UserCredentialsListResponse: + session = trans.sa_session + + source_type, source_id = payload.source_type, payload.source_id + + db_user_credentials = self._user_credentials( + trans, + user_id=user_id, + source_type=source_type, + source_id=source_id, + ) + credentials_dict = self._user_credentials_to_dict(db_user_credentials).values() + existing_groups = { + cred["reference"]: {group["name"]: group["id"] for group in cred["groups"].values()} + for cred in credentials_dict + } + + for service_payload in payload.credentials: + reference = service_payload.reference + current_group_name = service_payload.current_group + current_group_id = existing_groups.get(reference, {}).get(current_group_name) + + user_credentials = next((cred[0] for cred in db_user_credentials if cred[0].reference == reference), None) + if not user_credentials: + if user_id == "current": + user_id = trans.user.id + user_credentials = UserCredentials( + user_id=user_id, + reference=reference, + source_type=source_type, + source_id=source_id, + ) + session.add(user_credentials) + session.flush() + user_credentials_id = user_credentials.id + + for group in service_payload.groups: + group_name = group.name + + credentials_group = next( + (group[1] for group in db_user_credentials if group[1].name == group_name), None + ) + if not credentials_group: + credentials_group = CredentialsGroup(name=group_name, user_credentials_id=user_credentials_id) + session.add(credentials_group) + session.flush() + user_credential_group_id = credentials_group.id + + if current_group_name == group_name: + current_group_id = user_credential_group_id + + user_vault = UserVaultWrapper(self._app.vault, trans.user) + for variable_payload in group.variables: + variable_name, variable_value = variable_payload.name, variable_payload.value + variable = next( + ( + var[2] + for var in db_user_credentials + if var[1].name == group_name and var[2].name == variable_name + ), + None, + ) + if variable: + variable.value = variable_value or "" + else: + variable = Variable( + user_credential_group_id=user_credential_group_id, + name=variable_name, + value=variable_value or "", + ) + session.add(variable) + for secret_payload in group.secrets: + secret_name, secret_value = secret_payload.name, secret_payload.value + + secret = next( + ( + sec[3] + for sec in db_user_credentials + if sec[1].name == group_name and sec[3].name == secret_name + ), + None, + ) + if secret: + secret.already_set = True if secret_value else False + else: + secret = Secret( + user_credential_group_id=user_credential_group_id, + name=secret_name, + already_set=True if secret_value else False, + ) + session.add(secret) + vault_ref = f"{source_type}|{source_id}|{reference}|{group_name}|{secret_name}" + user_vault.write_secret(vault_ref, secret_value or "") + if not current_group_id: + raise exceptions.RequestParameterInvalidException( + "No group was selected as the current group.", type="error" + ) + user_credentials.current_group_id = current_group_id + session.add(user_credentials) + + with transaction(session): + session.commit() + + new_user_credentials = db_user_credentials or self._user_credentials( + trans, + user_id=user_id, + source_type=source_type, + source_id=source_id, + ) + credentials_dict = self._user_credentials_to_dict(new_user_credentials).values() + return UserCredentialsListResponse(root=[UserCredentialsResponse(**cred) for cred in credentials_dict]) + + def _delete_credentials(self, sa_session: galaxy_scoped_session, rows_to_be_deleted: List): + for row in rows_to_be_deleted: + sa_session.delete(row) + with transaction(sa_session): + sa_session.commit() diff --git a/test/functional/tools/secret_tool.xml b/test/functional/tools/secret_tool.xml new file mode 100644 index 000000000000..998bf14dcb1a --- /dev/null +++ b/test/functional/tools/secret_tool.xml @@ -0,0 +1,15 @@ + + + + + + + + + '$output' && echo \$service1_user >> '$output' && echo \$service1_pass >> '$output' + ]]> + + + + diff --git a/test/integration/test_credentials.py b/test/integration/test_credentials.py new file mode 100644 index 000000000000..ca383b8cc137 --- /dev/null +++ b/test/integration/test_credentials.py @@ -0,0 +1,76 @@ +from galaxy_test.driver import integration_util + + +class TestCredentialsApi(integration_util.IntegrationTestCase, integration_util.ConfiguresDatabaseVault): + @classmethod + def handle_galaxy_config_kwds(cls, config): + super().handle_galaxy_config_kwds(config) + cls._configure_database_vault(config) + + def test_provide_credential(self): + created_user_credentials = self._populate_user_credentials() + assert created_user_credentials[0]["current_group_name"] == "default" + + def test_list_user_credentials(self): + response = self._get("/api/users/current/credentials") + self._assert_status_code_is(response, 200) + list_user_credentials = response.json() + assert len(list_user_credentials) > 0 + + def test_delete_service_credentials(self): + created_user_credentials = self._populate_user_credentials() + user_credentials_id = created_user_credentials[0]["id"] + response = self._delete(f"/api/users/current/credentials/{user_credentials_id}") + self._assert_status_code_is(response, 204) + + def test_delete_credentials(self): + created_user_credentials = self._populate_user_credentials() + user_credentials_id = created_user_credentials[0]["id"] + group_id = list(created_user_credentials[0]["groups"].values())[0]["id"] + response = self._delete(f"/api/users/current/credentials/{user_credentials_id}/{group_id}") + self._assert_status_code_is(response, 204) + + def test_invalid_provide_credential(self): + payload = { + "source_type": "tool", + "source_id": "test_tool", + "credentials": [ + { + "reference": "invalid_test_service", + "current_group": "invalid_group_name", + "groups": [{"name": "default", "variables": [], "secrets": []}], + } + ], + } + response = self._post("/api/users/current/credentials", data=payload, json=True) + self._assert_status_code_is(response, 400) + + def test_delete_not_existing_service_credentials(self): + response = self._delete("/api/users/current/credentials/f2db41e1fa331b3e") + self._assert_status_code_is(response, 400) + + def test_delete_not_existing_credentials(self): + response = self._delete("/api/users/current/credentials/f2db41e1fa331b3e/f2db41e1fa331b3e") + self._assert_status_code_is(response, 400) + + def _populate_user_credentials(self): + payload = { + "source_type": "tool", + "source_id": "test_tool", + "credentials": [ + { + "reference": "test_service", + "current_group": "default", + "groups": [ + { + "name": "default", + "variables": [{"name": "username", "value": "user"}], + "secrets": [{"name": "password", "value": "pass"}], + } + ], + } + ], + } + response = self._post("/api/users/current/credentials", data=payload, json=True) + self._assert_status_code_is(response, 200) + return response.json() diff --git a/test/unit/app/tools/test_evaluation.py b/test/unit/app/tools/test_evaluation.py index e571b5fd8898..72fe91c54b94 100644 --- a/test/unit/app/tools/test_evaluation.py +++ b/test/unit/app/tools/test_evaluation.py @@ -306,6 +306,7 @@ def __init__(self, app): self.options = Bunch(sanitize=False) self.check_values = True self.version_string_cmd = "" + self.credentials = [] def test_thresh_param(self): elem = XML('') diff --git a/test/unit/tool_util/test_cwl.py b/test/unit/tool_util/test_cwl.py index a3bae657b132..f9d48cd503b3 100644 --- a/test/unit/tool_util/test_cwl.py +++ b/test/unit/tool_util/test_cwl.py @@ -281,7 +281,7 @@ def test_load_proxy_simple(): outputs, output_collections = tool_source.parse_outputs(None) assert len(outputs) == 1 - software_requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers() + software_requirements, containers, resource_requirements, credentials = tool_source.parse_requirements() assert software_requirements.to_dict() == [] assert len(containers) == 1 assert containers[0].to_dict() == { @@ -292,6 +292,7 @@ def test_load_proxy_simple(): } assert len(resource_requirements) == 1 assert resource_requirements[0].to_dict() == {"resource_type": "ram_min", "value_or_expression": 8} + assert len(credentials) == 0 def test_representation_id(): diff --git a/test/unit/tool_util/test_parsing.py b/test/unit/tool_util/test_parsing.py index 21ec92387694..16991d3129d7 100644 --- a/test/unit/tool_util/test_parsing.py +++ b/test/unit/tool_util/test_parsing.py @@ -50,6 +50,11 @@ 1 2 67108864 + + + + + @@ -159,6 +164,24 @@ containers: - type: docker identifier: "awesome/bowtie" +credentials: + - name: Apollo + reference: gmod.org/apollo + optional: true + secrets: + - name: username + label: Your Apollo username + description: Username for Apollo + inject_as_env: apollo_user + - name: password + label: Your Apollo password + description: Password for Apollo + inject_as_env: apollo_pass + variables: + - name: server + label: Your Apollo server + description: URL of your Apollo server + inject_as_env: apollo_url outputs: out1: format: bam @@ -347,7 +370,7 @@ def test_action(self): assert self._tool_source.parse_action_module() is None def test_requirements(self): - requirements, containers, resource_requirements = self._tool_source.parse_requirements_and_containers() + requirements, containers, resource_requirements, credentials = self._tool_source.parse_requirements() assert requirements[0].type == "package" assert list(containers)[0].identifier == "mycool/bwa" assert resource_requirements[0].resource_type == "cores_min" @@ -358,6 +381,11 @@ def test_requirements(self): assert resource_requirements[5].resource_type == "cuda_device_count_max" assert resource_requirements[6].resource_type == "shm_size" assert not resource_requirements[0].runtime_required + assert credentials[0].name == "Apollo" + assert credentials[0].reference == "gmod.org/apollo" + assert credentials[0].optional + assert len(credentials[0].secrets) == 2 + assert len(credentials[0].variables) == 1 def test_outputs(self): outputs, output_collections = self._tool_source.parse_outputs(object()) @@ -533,7 +561,7 @@ def test_action(self): assert self._tool_source.parse_action_module() is None def test_requirements(self): - software_requirements, containers, resource_requirements = self._tool_source.parse_requirements_and_containers() + software_requirements, containers, resource_requirements, credentials = self._tool_source.parse_requirements() assert software_requirements.to_dict() == [{"name": "bwa", "type": "package", "version": "1.0.1", "specs": []}] assert len(containers) == 1 assert containers[0].to_dict() == { @@ -562,6 +590,9 @@ def test_requirements(self): "resource_type": "shm_size", "value_or_expression": 67108864, } + assert len(credentials) == 1 + assert len(credentials[0].secrets) == 2 + assert len(credentials[0].variables) == 1 def test_outputs(self): outputs, output_collections = self._tool_source.parse_outputs(object())