merge: Enable security scanning in CI pipelines (resolves #407, replaces !414) (!1232)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/1232

Closes #407

Approved-by: dakkar <dakkar@thenautilus.net>
Approved-by: Marie <github@yuugi.dev>

.gitlab-ci.yml

@@ -1,13 +1,73 @@
 stages:
+  - build
   - test
   - deploy
 
-.test_common: &test_common
+variables:
-  stage: test
+  # https://docs.gitlab.com/user/application_security/sast/gitlab_advanced_sast
-  image: docker.io/node:22
+  GITLAB_ADVANCED_SAST_ENABLED: 'true'
+
+  # https://docs.gitlab.com/user/application_security/sast/#vulnerability-filters
+  # https://stackoverflow.com/a/71111784
+  SAST_EXCLUDED_PATHS: 'spec,test,test-d,test-federation,test-server,tests,tmp,cypress,coverage,node_modules,build,built,built-js,*.min.js,megalodon/lib,libopenmpt'
+  DS_EXCLUDED_PATHS: 'spec,test,test-d,test-federation,test-server,tests,tmp,cypress,coverage,node_modules,build,built,built-js,*.min.js,megalodon/lib,libopenmpt,packages/*/src' # save time: skip source directories
+
+  # https://docs.gitlab.com/user/application_security/dependency_scanning/migration_guide_to_sbom_based_scans/
+  DS_ENFORCE_NEW_ANALYZER: 'true'
+  DS_MAX_DEPTH: -1
+  # https://docs.gitlab.com/user/application_security/dependency_scanning/static_reachability/
+  DS_STATIC_REACHABILITY_ENABLED: true
+
+  # https://docs.gitlab.com/user/application_security/detect/security_configuration/#use-security-scanning-tools-with-merge-request-pipelines
+  AST_ENABLE_MR_PIPELINES: 'true'
+
+.common: &common
+  # "only" has been removed, so we use rules.
+  # This runs in MR pipelines *or* push to develop/stable
+  rules: &common-rules
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: $CI_PIPELINE_SOURCE == 'push' && ($CI_COMMIT_BRANCH == 'develop' || $CI_COMMIT_BRANCH == 'stable')
+
+.deploy_common: &deploy_common
+  stage: deploy
+  # Only run when pushing to stable, develop, or tags
+  rules: &deploy-rules
+    - if: $CI_PIPELINE_SOURCE != 'push'
+      when: never
+    - if: $CI_COMMIT_BRANCH == 'develop'
+    - if: $CI_COMMIT_BRANCH == 'stable'
+    - if: $CI_COMMIT_TAG
+
+# https://docs.gitlab.com/user/application_security/sast/
+include:
+  - local: '.gitlab/ci_templates/dependency_scanning.yml'
+    rules: *common-rules
+  - local: '.gitlab/ci_templates/container_scanning.yml'
+    rules: *deploy-rules
+  - local: '.gitlab/ci_templates/sast.yml'
+    rules: *common-rules
+  - local: '.gitlab/ci_templates/secret_detection.yml'
+    rules: *common-rules
+  - local: '.gitlab/ci_templates/lib_behave.yml'
+    rules: *common-rules
+
+# Cache node_modules and share build artifacts for the pipeline.
+# This shares the same cache definition, but it's the only place that actually *pushes* to the cache.
+# https://docs.gitlab.com/ci/caching/
+# https://github.com/pnpm/pnpm/issues/1174#issuecomment-996719439
+# https://github.com/pnpm/pnpm/issues/1174#issuecomment-1641267133
+build: &build
+  <<: *common
+  stage: build
+  image:
+    name: docker.io/node:22
+    pull_policy: if-not-present
   variables:
-    POSTGRES_PASSWORD: ci
+    POSTGRES_PASSWORD: 'ci'
-    COREPACK_DEFAULT_TO_LATEST: 0
+    COREPACK_DEFAULT_TO_LATEST: '0'
+  # Arm64 is recommended for CI
+  tags:
+    - arm64
   before_script:
     - apt-get update && apt-get install -y git wget curl build-essential python3 ffmpeg libcairo2-dev libpango1.0-dev libpangocairo-1.0
     - 'echo "clusterLimit: $(nproc)" >> .config/ci.yml'
@@ -16,30 +76,57 @@ stages:
     - corepack enable
     - corepack install
     - git submodule update --init
+    - pnpm config set store-dir .pnpm-store
     - pnpm install --frozen-lockfile
+  script:
+    - pnpm run build
   cache:
-    key: test
+    - &cache-pnpm
+      key:
+        files:
+          - 'pnpm-lock.yaml'
+          - 'pnpm-workspace.yaml'
+      paths:
+        - '.pnpm-store/'
+        - 'node_modules/'
+        - 'packages/*/node_modules/'
+        - 'packages/misskey-js/generator/node_modules/'
       policy: pull-push
       when: on_success
+    - &cache-build
+      key: "$CI_COMMIT_REF_SLUG"
       paths:
-     - node_modules/
+        - 'built/'
-     - packages/*/node_modules/
+        - 'packages/*/built/'
-  only:
+        - 'packages/megalodon/lib/'
-    - develop
+      policy: pull-push
-    - merge_requests
+      when: on_success
-    - stable
+
+.test_common: &test_common
+  <<: *common
+  <<: *build
+  stage: test
+  cache:
+    -
+      <<: *cache-pnpm
+      policy: pull
+    -
+      <<: *cache-build
+      policy: pull
 
 lint:
   <<: *test_common
   script:
-    - pnpm run build
+    - pnpm run build-assets
     - pnpm run eslint
 
 backend_tests:
   <<: *test_common
   services:
-    - postgres:15
+    - name: postgres:15
-    - redis
+      pull_policy: if-not-present
+    - name: redis
+      pull_policy: if-not-present
   script:
     - >-
       pnpm run build \
@@ -48,6 +135,19 @@ backend_tests:
         --filter=misskey-js
     - pnpm run migrate
     - pnpm run test --filter=backend
+  # Same as common, but MRs are only run if they modify the backend.
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'push' && ($CI_COMMIT_BRANCH == 'develop' || $CI_COMMIT_BRANCH == 'stable')
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        - 'packages/backend/**/*'
+        - 'packages/megalodon/**/*'
+        - 'packages/misskey-js/**/*'
+        - 'packages/*' # single-star is intention - we don't want to recurse!
+        - 'scripts/**/*'
+        - 'eslint/**/*'
+        - 'chart/**/*'
+        - '.config/**/*'
 
 frontend_tests:
   <<: *test_common
@@ -57,13 +157,36 @@ frontend_tests:
         --filter=frontend \
         --filter=frontend-embed \
         --filter=frontend-shared \
-        --filter=megalogon \
+        --filter=misskey-js \
-        --filter=misskey-js
+        --filter=misskey-bubble-game \
+        --filter=misskey-reversi \
+        --filter=sw
     - pnpm run test --filter=frontend --filter=misskey-js
+  # Same as common, but MRs are only run if they modify the frontend.
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'push' && ($CI_COMMIT_BRANCH == 'develop' || $CI_COMMIT_BRANCH == 'stable')
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      changes:
+        - 'packages/frontend/**/*'
+        - 'packages/frontend-embed/**/*'
+        - 'packages/frontend-shared/**/*'
+        - 'packages/misskey-js/**/*'
+        - 'packages/misskey-bubble-game/**/*'
+        - 'packages/misskey-reversi/**/*'
+        - 'packages/sw/**/*'
+        - 'packages/*' # single-star is intention - we don't want to recurse!
+        - 'scripts/**/*'
+        - 'eslint/**/*'
+        - 'locales/**/*'
+        - 'sharkey-locales/**/*'
+        - 'cypress/**/*'
+        - 'assets/**/*'
 
 get_image_tag:
-  stage: deploy
+  <<: *deploy_common
-  image: docker.io/alpine:latest
+  image:
+    name: docker.io/alpine:latest
+    pull_policy: if-not-present
   script:
     - apk add jq
     - |
@@ -82,13 +205,9 @@ get_image_tag:
   artifacts:
     reports:
       dotenv: build.env
-  only:
-    - stable
-    - develop
-    - tags
 
 build_image:
-  stage: deploy
+  <<: *deploy_common
   needs:
     - job: get_image_tag
       artifacts: true
@@ -100,6 +219,7 @@ build_image:
     - ${ARCH}
   image:
     name: gcr.io/kaniko-project/executor:debug
+    pull_policy: if-not-present
     entrypoint: [""]
   script:
     - >-
@@ -108,13 +228,9 @@ build_image:
         --dockerfile "${CI_PROJECT_DIR}/Dockerfile" \
         --single-snapshot \
         --destination "${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_VERSION}-${ARCH}"
-  only:
-    - stable
-    - develop
-    - tags
 
 merge_image_manifests:
-  stage: deploy
+  <<: *deploy_common
   needs:
     - job: build_image
       artifacts: false
@@ -122,6 +238,7 @@ merge_image_manifests:
       artifacts: true
   image:
     name: mplatform/manifest-tool:alpine
+    pull_policy: if-not-present
     entrypoint: [""]
   script:
     - >-
@@ -133,7 +250,3 @@ merge_image_manifests:
         --tags ${REGISTRY_PUSH_VERSION} \
         --template ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_VERSION}-ARCH \
         --target ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG}
-  only:
-    - stable
-    - develop
-    - tags

.gitlab/ci_templates/container_scanning.yml

@@ -0,0 +1,20 @@
+# https://docs.gitlab.com/user/application_security/sast/
+include:
+  - template: Jobs/Container-Scanning.latest.gitlab-ci.yml
+
+# https://docs.gitlab.com/user/application_security/container_scanning/#scanning-archives-built-in-a-previous-job
+# https://docs.gitlab.com/user/application_security/detect/security_configuration/#error-chosen-stage-test-does-not-exist
+container_scanning:
+  stage: deploy
+
+  # SAST tools only support x64
+  tags:
+    - amd64
+
+  variables:
+    AST_ENABLE_MR_PIPELINES: 'false'
+    CS_IMAGE: "${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG}"
+
+  needs:
+    - job: merge_image_manifests
+      artifacts: true

.gitlab/ci_templates/dependency_scanning.yml

@@ -0,0 +1,14 @@
+# https://docs.gitlab.com/user/application_security/sast/
+include:
+  - template: Jobs/Dependency-Scanning.latest.gitlab-ci.yml
+
+dependency-scanning:
+  stage: test
+
+  # SAST tools only support x64
+  tags:
+    - amd64
+
+  # Don't wait, since this has no dependencies.
+  # https://docs.gitlab.com/ci/yaml/#needs
+  needs: []

.gitlab/ci_templates/lib_behave.yml

@@ -0,0 +1,20 @@
+# https://docs.gitlab.com/user/application_security/sast/
+include:
+  # https://docs.gitlab.com/user/application_security/dependency_scanning/experiment_libbehave_dependency/
+  - component: $CI_SERVER_FQDN/TransFem-org/libbehave/libbehave@v0.4.0
+    inputs:
+      include-lang: 'js'
+      stage: test
+
+
+.libbehave-experiment:
+  # SAST tools only support x64
+  tags:
+    - amd64
+
+  # Don't wait, since this has no dependencies.
+  # https://docs.gitlab.com/ci/yaml/#needs
+  needs: []
+
+  # Gitlab issue currently causes error when upload final artifacts
+  allow_failure: true

.gitlab/ci_templates/sast.yml

@@ -0,0 +1,17 @@
+# https://docs.gitlab.com/user/application_security/sast/
+include:
+  - template: Jobs/SAST.latest.gitlab-ci.yml
+
+sast: &sast
+  stage: test
+
+  # SAST tools only support x64
+  tags:
+    - amd64
+
+  # Don't wait, since this has no dependencies.
+  # https://docs.gitlab.com/ci/yaml/#needs
+  needs: []
+
+gitlab-advanced-sast:
+  <<: *sast

.gitlab/ci_templates/secret_detection.yml

@@ -0,0 +1,14 @@
+# https://docs.gitlab.com/user/application_security/sast/
+include:
+  - template: Jobs/Secret-Detection.latest.gitlab-ci.yml
+
+secret_detection:
+  stage: test
+
+  # SAST tools only support x64
+  tags:
+    - amd64
+
+  # Don't wait, since this has no dependencies.
+  # https://docs.gitlab.com/ci/yaml/#needs
+  needs: []

packages/misskey-js/etc/misskey-js.api.md

@@ -347,6 +347,12 @@ type AdminResetPasswordResponse = operations['admin___reset-password']['response
 // @public (undocumented)
 type AdminResolveAbuseUserReportRequest = operations['admin___resolve-abuse-user-report']['requestBody']['content']['application/json'];
 
+// @public (undocumented)
+type AdminRolesAnnotateConditionRequest = operations['admin___roles___annotate-condition']['requestBody']['content']['application/json'];
+
+// @public (undocumented)
+type AdminRolesAnnotateConditionResponse = operations['admin___roles___annotate-condition']['responses']['200']['content']['application/json'];
+
 // @public (undocumented)
 type AdminRolesAssignRequest = operations['admin___roles___assign']['requestBody']['content']['application/json'];
 
@@ -1622,6 +1628,8 @@ declare namespace entities {
         AdminResetPasswordRequest,
         AdminResetPasswordResponse,
         AdminResolveAbuseUserReportRequest,
+        AdminRolesAnnotateConditionRequest,
+        AdminRolesAnnotateConditionResponse,
         AdminRolesAssignRequest,
         AdminRolesCloneRequest,
         AdminRolesCloneResponse,

packages/misskey-js/src/autogen/apiClientJSDoc.ts

@@ -867,6 +867,17 @@ declare module '../api.js' {
       credential?: string | null,
     ): Promise<SwitchCaseResponseType<E, P>>;
 
+    /**
+     * No description provided.
+     * 
+     * **Credential required**: *Yes* / **Permission**: *read:admin:roles*
+     */
+    request<E extends 'admin/roles/annotate-condition', P extends Endpoints[E]['req']>(
+      endpoint: E,
+      params: P,
+      credential?: string | null,
+    ): Promise<SwitchCaseResponseType<E, P>>;
+
     /**
      * No description provided.
      * 

packages/misskey-js/src/autogen/endpoint.ts

@@ -103,6 +103,8 @@ import type {
 	AdminResetPasswordRequest,
 	AdminResetPasswordResponse,
 	AdminResolveAbuseUserReportRequest,
+	AdminRolesAnnotateConditionRequest,
+	AdminRolesAnnotateConditionResponse,
 	AdminRolesAssignRequest,
 	AdminRolesCloneRequest,
 	AdminRolesCloneResponse,
@@ -748,6 +750,7 @@ export type Endpoints = {
 	'admin/relays/remove': { req: AdminRelaysRemoveRequest; res: EmptyResponse };
 	'admin/reset-password': { req: AdminResetPasswordRequest; res: AdminResetPasswordResponse };
 	'admin/resolve-abuse-user-report': { req: AdminResolveAbuseUserReportRequest; res: EmptyResponse };
+	'admin/roles/annotate-condition': { req: AdminRolesAnnotateConditionRequest; res: AdminRolesAnnotateConditionResponse };
 	'admin/roles/assign': { req: AdminRolesAssignRequest; res: EmptyResponse };
 	'admin/roles/clone': { req: AdminRolesCloneRequest; res: AdminRolesCloneResponse };
 	'admin/roles/create': { req: AdminRolesCreateRequest; res: AdminRolesCreateResponse };

packages/misskey-js/src/autogen/entities.ts

@@ -106,6 +106,8 @@ export type AdminRelaysRemoveRequest = operations['admin___relays___remove']['re
 export type AdminResetPasswordRequest = operations['admin___reset-password']['requestBody']['content']['application/json'];
 export type AdminResetPasswordResponse = operations['admin___reset-password']['responses']['200']['content']['application/json'];
 export type AdminResolveAbuseUserReportRequest = operations['admin___resolve-abuse-user-report']['requestBody']['content']['application/json'];
+export type AdminRolesAnnotateConditionRequest = operations['admin___roles___annotate-condition']['requestBody']['content']['application/json'];
+export type AdminRolesAnnotateConditionResponse = operations['admin___roles___annotate-condition']['responses']['200']['content']['application/json'];
 export type AdminRolesAssignRequest = operations['admin___roles___assign']['requestBody']['content']['application/json'];
 export type AdminRolesCloneRequest = operations['admin___roles___clone']['requestBody']['content']['application/json'];
 export type AdminRolesCloneResponse = operations['admin___roles___clone']['responses']['200']['content']['application/json'];

packages/misskey-js/src/autogen/types.ts

@@ -720,6 +720,15 @@ export type paths = {
      */
     post: operations['admin___resolve-abuse-user-report'];
   };
+  '/admin/roles/annotate-condition': {
+    /**
+     * admin/roles/annotate-condition
+     * @description No description provided.
+     *
+     * **Credential required**: *Yes* / **Permission**: *read:admin:roles*
+     */
+    post: operations['admin___roles___annotate-condition'];
+  };
   '/admin/roles/assign': {
     /**
      * admin/roles/assign
@@ -10521,6 +10530,63 @@ export type operations = {
       };
     };
   };
+  /**
+   * admin/roles/annotate-condition
+   * @description No description provided.
+   *
+   * **Credential required**: *Yes* / **Permission**: *read:admin:roles*
+   */
+  'admin___roles___annotate-condition': {
+    requestBody: {
+      content: {
+        'application/json': {
+          /** Format: misskey:id */
+          userId: string;
+          condFormula: Record<string, never>;
+        };
+      };
+    };
+    responses: {
+      /** @description OK (with results) */
+      200: {
+        content: {
+          'application/json': {
+            [key: string]: boolean;
+          };
+        };
+      };
+      /** @description Client error */
+      400: {
+        content: {
+          'application/json': components['schemas']['Error'];
+        };
+      };
+      /** @description Authentication error */
+      401: {
+        content: {
+          'application/json': components['schemas']['Error'];
+        };
+      };
+      /** @description Forbidden error */
+      403: {
+        content: {
+          'application/json': components['schemas']['Error'];
+        };
+      };
+      /** @description I'm Ai */
+      418: {
+        content: {
+          'application/json': components['schemas']['Error'];
+        };
+      };
+      /** @description Internal server error */
+      500: {
+        content: {
+          'application/json': components['schemas']['Error'];
+        };
+      };
+    };
+  };
   /**
    * admin/roles/assign
    * @description No description provided.