[M90] Generate the LUCI services configs into a luci subdirectory.

lucicfg validate uploads all of the files that are present in the
projects config directory, which includes the builders subdirectory.
When files are generated for each builder using the chromium family of
recipes, this causes lucicfg validate to time out. Changing the config
directory of the project to luci will put all of the LUCI services
configs in a separate directory so that only those are uploaded to the
luci-config service, which will remove any of our custom files from
being uploaded.

Bug: 1245674
Change-Id: I89ce93a680d2f25c0f254a4d5babd0aa4c6d9211
No-Try: true
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3138095
Commit-Queue: Garrett Beaty <gbeaty@google.com>
Reviewed-by: Ben Pastene <bpastene@chromium.org>
Cr-Commit-Position: refs/branch-heads/4430@{#1577}
Cr-Branched-From: e5ce7dc-refs/heads/master@{#857950}

infra/config/PRESUBMIT.py

@@ -12,8 +12,8 @@
 
 
 def CheckLintLuciMilo(input_api, output_api):
-  if ('infra/config/generated/luci-milo.cfg' in input_api.LocalPaths() or
-      'infra/config/lint-luci-milo.py' in input_api.LocalPaths()):
+  if ('infra/config/generated/luci/luci-milo.cfg' in input_api.LocalPaths()
+      or 'infra/config/lint-luci-milo.py' in input_api.LocalPaths()):
     return input_api.RunTests([
         input_api.Command(
             name='lint-luci-milo',
@@ -24,8 +24,9 @@ def CheckLintLuciMilo(input_api, output_api):
   return []
 
 def CheckTestingBuildbot(input_api, output_api):
-  if ('infra/config/generated/luci-milo.cfg' in input_api.LocalPaths() or
-      'infra/config/generated/luci-milo-dev.cfg' in input_api.LocalPaths()):
+  if ('infra/config/generated/luci/luci-milo.cfg' in input_api.LocalPaths() or
+      'infra/config/generated/luci/luci-milo-dev.cfg' in input_api.LocalPaths()
+      ):
     return input_api.RunTests([
         input_api.Command(
             name='testing/buildbot config checks',

infra/config/README.md

@@ -4,7 +4,7 @@ https://chrome-internal.googlesource.com/infradata/config/+/refs/heads/master/co
 
 This directory contains chromium project-wide configurations
 for Chrome Operations services.
-For example, [cr-buildbucket.cfg](generated/cr-buildbucket.cfg) defines
+For example, [cr-buildbucket.cfg](generated/luci/cr-buildbucket.cfg) defines
 builders.
 
 Currently active version can be checked at

infra/config/console-header.star

@@ -299,7 +299,7 @@ HEADER = headers.header(
                 headers.link(
                     text = "customize",
                     branch_selector = branches.ALL_BRANCHES,
-                    url = "https://chromium.googlesource.com/chromium/src/+/{}/infra/config/generated/luci-milo.cfg".format(settings.ref),
+                    url = "https://chromium.googlesource.com/chromium/src/+/{}/infra/config/generated/luci/luci-milo.cfg".format(settings.ref),
                     alt = "Customize this console",
                 ),
             ],

infra/config/dev.star

@@ -20,11 +20,11 @@ lucicfg.enable_experiment("crbug.com/1085650")
 lucicfg.config(
     config_dir = "generated",
     tracked_files = [
-        "cr-buildbucket-dev.cfg",
-        "luci-logdog-dev.cfg",
-        "luci-milo-dev.cfg",
-        "luci-scheduler-dev.cfg",
-        "realms-dev.cfg",
+        "luci/cr-buildbucket-dev.cfg",
+        "luci/luci-logdog-dev.cfg",
+        "luci/luci-milo-dev.cfg",
+        "luci/luci-scheduler-dev.cfg",
+        "luci/realms-dev.cfg",
     ],
     fail_on_warnings = True,
 )

infra/config/dev/chromium-header.textpb

@@ -93,7 +93,7 @@ links: {
   }
   links: {
     text: "customize"
-    url: "https://chromium.googlesource.com/chromium/src/+/master/infra/config/luci-milo-dev.cfg"
+    url: "https://chromium.googlesource.com/chromium/src/+/HEAD/infra/config/generated/luci/luci-milo-dev.cfg"
       alt: "Customize this console"
   }
 }

infra/config/dev/dev.star

@@ -7,6 +7,7 @@
 
 luci.project(
     name = "chromium",
+    config_dir = "luci",
     dev = True,
     buildbucket = "cr-buildbucket-dev.appspot.com",
     logdog = "luci-logdog-dev.appspot.com",

infra/config/generators/cq-builders-md.star

@@ -49,7 +49,7 @@ _TRY_BUILDER_VIEW_URL = "https://ci.chromium.org/p/chromium/builders/try"
 _REGEX_PREFIX = ".+/[+]/"
 
 def _get_main_config_group_builders(ctx):
-    cq_cfg = ctx.output["commit-queue.cfg"]
+    cq_cfg = ctx.output["luci/commit-queue.cfg"]
 
     for c in cq_cfg.config_groups:
         if len(c.gerrit) != 1:

infra/config/generators/scheduler-noop-jobs.star

@@ -61,7 +61,7 @@ _TESTER_NOOP_JOBS = [scheduler_pb.Job(
 def _add_noop_jobs(ctx):
     if settings.is_main:
         return
-    cfg = ctx.output["luci-scheduler.cfg"]
+    cfg = ctx.output["luci/luci-scheduler.cfg"]
     for j in _TESTER_NOOP_JOBS:
         cfg.job.append(j)
 

infra/config/generators/sort-consoles.star

@@ -1,5 +1,5 @@
 def _sort_consoles(ctx):
-    milo = ctx.output["luci-milo.cfg"]
+    milo = ctx.output["luci/luci-milo.cfg"]
 
     # Sort so that the overview consoles appear at the top of the console list
     # The overview consoles specify a title, so checking c.id == c.name will put

infra/config/lib/consoles.star

@@ -392,7 +392,7 @@ def list_view(*, name, branch_selector = branches.MAIN, **kwargs):
     )
 
 def _sort_consoles(ctx):
-    milo = ctx.output["luci-milo.cfg"]
+    milo = ctx.output["luci/luci-milo.cfg"]
     consoles = []
     for console in milo.consoles:
         if not console.builders:

infra/config/lint-luci-milo.py

@@ -55,7 +55,8 @@ def to_list(builders, category_prefix=''):
 
 def main():
   project = project_pb2.Project()
-  with open(os.path.join(THIS_DIR, 'generated', 'luci-milo.cfg'), 'rb') as f:
+  with open(os.path.join(THIS_DIR, 'generated', 'luci', 'luci-milo.cfg'),
+            'rb') as f:
     google.protobuf.text_format.Parse(f.read(), project,
                                       allow_unknown_field=True)
 

infra/config/main.star

@@ -21,19 +21,19 @@ lucicfg.enable_experiment("crbug.com/1085650")
 lucicfg.config(
     config_dir = "generated",
     tracked_files = [
-        "commit-queue.cfg",
         "cq-builders.md",
-        "cr-buildbucket.cfg",
-        "luci-logdog.cfg",
-        "luci-milo.cfg",
-        "luci-notify.cfg",
-        "luci-notify/email-templates/*.template",
-        "luci-scheduler.cfg",
+        "luci/commit-queue.cfg",
+        "luci/cr-buildbucket.cfg",
+        "luci/luci-logdog.cfg",
+        "luci/luci-milo.cfg",
+        "luci/luci-notify.cfg",
+        "luci/luci-notify/email-templates/*.template",
+        "luci/luci-scheduler.cfg",
+        "luci/project.cfg",
+        "luci/realms.cfg",
+        "luci/tricium-prod.cfg",
         "outages.pyl",
-        "project.cfg",
         "project.pyl",
-        "realms.cfg",
-        "tricium-prod.cfg",
     ],
     fail_on_warnings = True,
     lint_checks = [
@@ -49,12 +49,13 @@ lucicfg.config(
 
 # Just copy tricium-prod.cfg to the generated outputs
 lucicfg.emit(
-    dest = "tricium-prod.cfg",
+    dest = "luci/tricium-prod.cfg",
     data = io.read_file("tricium-prod.cfg"),
 )
 
 luci.project(
     name = settings.project,
+    config_dir = "luci",
     buildbucket = "cr-buildbucket.appspot.com",
     logdog = "luci-logdog.appspot.com",
     milo = "luci-milo.appspot.com",

infra/config/outages/outages.star

@@ -24,7 +24,7 @@ def _disable_cq_experiments(ctx):
     if not config.disable_cq_experiments:
         return
 
-    for c in ctx.output["commit-queue.cfg"].config_groups:
+    for c in ctx.output["luci/commit-queue.cfg"].config_groups:
         if c.verifiers.tryjob == cq_pb.Verifiers.Tryjob():
             # Accessing the tryjob field where it wasn't set causes it to be set
             # to an empty message and added to the output

infra/config/subprojects/chromium/fallback-cq.star

@@ -34,7 +34,7 @@ luci.cq_group(
 # the empty CQ group until it's exposed in lucicfg or there is a better way to
 # create a CQ group for all of the canary branches
 def _generate_cq_group_fallback(ctx):
-    cq_cfg = ctx.output["commit-queue.cfg"]
+    cq_cfg = ctx.output["luci/commit-queue.cfg"]
 
     for c in cq_cfg.config_groups:
         if c.name == "fallback-empty-cq":

infra/config/validators/builders-in-consoles.star

@@ -5,15 +5,15 @@
 def _validate_builders_in_console(ctx):
     builders = {}
 
-    for console in ctx.output["luci-milo.cfg"].consoles:
+    for console in ctx.output["luci/luci-milo.cfg"].consoles:
         for builder in console.builders:
             _, long_bucket, builder_name = builder.name.split("/")
             _, _, bucket = long_bucket.split(".", 2)
             builders.setdefault(bucket, {})[builder_name] = True
 
     builders_without_console = []
 
-    for bucket in ctx.output["cr-buildbucket.cfg"].buckets:
+    for bucket in ctx.output["luci/cr-buildbucket.cfg"].buckets:
         bucket_builders = builders.get(bucket.name, {})
         for builder in bucket.swarming.builders:
             if builder.name not in bucket_builders:

testing/buildbot/generate_buildbot_json.py

@@ -1422,7 +1422,7 @@ def write_json_result(self, result): # pragma: no cover
       self.write_file(self.pyl_file_path(filename + suffix), jsonstr)
 
   def get_valid_bot_names(self):
-    # Extract bot names from infra/config/generated/luci-milo.cfg.
+    # Extract bot names from infra/config/generated/luci/luci-milo.cfg.
     # NOTE: This reference can cause issues; if a file changes there, the
     # presubmit here won't be run by default. A manually maintained list there
     # tries to run presubmit here when luci-milo.cfg is changed. If any other
@@ -1442,8 +1442,13 @@ def get_valid_bot_names(self):
         return None
 
     bot_names = set()
+    # TODO(https://crbug.com/1245674) Remove the glob without the luci component
+    # once all configs are switched over
     milo_configs = glob.glob(
         os.path.join(self.args.infra_config_dir, 'generated', 'luci-milo*.cfg'))
+    milo_configs += glob.glob(
+        os.path.join(self.args.infra_config_dir, 'generated', 'luci',
+                     'luci-milo*.cfg'))
     for c in milo_configs:
       for l in self.read_file(c).splitlines():
         if (not 'name: "buildbucket/luci.chromium.' in l and

testing/buildbot/generate_buildbot_json_unittest.py

@@ -74,8 +74,8 @@ def __init__(self,
         (pyl_files_dir, 'gn_isolate_map2.pyl'): GPU_TELEMETRY_GN_ISOLATE_MAP,
         (pyl_files_dir, 'variants.pyl'): variants,
         (infra_config_dir, 'generated/project.pyl'): project_pyl,
-        (infra_config_dir, 'generated/luci-milo.cfg'): luci_milo_cfg,
-        (infra_config_dir, 'generated/luci-milo-dev.cfg'): '',
+        (infra_config_dir, 'generated/luci/luci-milo.cfg'): luci_milo_cfg,
+        (infra_config_dir, 'generated/luci/luci-milo-dev.cfg'): '',
     }
     for (d, filename), content in files.iteritems():
       if content is None: