Add run_clean_db_request to clean db

Signed-off-by: Magnus Granberg <zorry@gentoo.org>

5 files changed, 183 insertions, 0 deletions

diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py
index b3ddb16..584dfdf 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -20,6 +20,24 @@ def getWorkers(worker_type, workers):
                 worker_list.append(worker['uuid'])
         return worker_list
 
+# check if we have same package_data
+@defer.inlineCallbacks
+def checkPackageData(master, builder, req1, req2):
+    canBeCollapsed = yield buildrequest.BuildRequest.canBeCollapsed(master, req1, req2)
+    selfBuildset , otherBuildset = yield defer.gatherResults([
+        master.data.get(('buildsets', req1['buildsetid'])),
+        master.data.get(('buildsets', req2['buildsetid']))
+    ])
+    print(f"TestCollapsed")
+    print(f"canBeCollapsed: {canBeCollapsed}")
+    print(f"selfBuildset: {selfBuildset}")
+    print(f"otherBuildset: {otherBuildset}")
+    if canBeCollapsed and selfBuildset['parent_buildid'] != None and \
+            otherBuildset['parent_buildid'] != None:
+        return True
+    else:
+        return False
+
 @defer.inlineCallbacks
 def CanWorkerBuildProject(builder, wfb, request):
     gentooci = builder.master.namedServices['services'].namedServices['gentooci']
@@ -118,4 +136,15 @@ def gentoo_builders(worker_data):
         factory=buildfactorys.run_build_images_request()
         )
     )
+    # Use multiplay workers
+    b.append(util.BuilderConfig(
+        name='run_clean_db_request',
+        workernames=getWorkersLocal('local', worker_data['local']),
+        # look builder so we only do one time
+        # look=
+        # if we allready runnin with same package_data properties then skip/collapse
+        #collapseRequests=checkPackageData,
+        factory=buildfactorys.run_clean_db_request()
+        )
+    )
     return b
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index b47a209..6755308 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -14,6 +14,7 @@ from buildbot_gentoo_ci.steps import portage
 from buildbot_gentoo_ci.steps import logs
 from buildbot_gentoo_ci.steps import repos
 from buildbot_gentoo_ci.steps import nodes
+from buildbot_gentoo_ci.steps import clean
 
 def update_db_check():
     f = util.BuildFactory()
@@ -66,6 +67,8 @@ def update_db_cpv():
     f.addStep(package.TriggerCheckForV())
     # update metadata if needed
     f.addStep(package.CheckMetadataPackagePath())
+    # clean package db if needed (clean up ebuilds in db)
+    f.addStep(package.TriggerCleanPackageDb())
     return f
 
 def update_db_v():
@@ -201,3 +204,9 @@ def run_build_images_request():
     # set the needed steps for making the image
     f.addStep(nodes.SetupBuildTypeAndSteps())
     return f
+
+def run_clean_db_request():
+    f = util.BuildFactory()
+    # set needed Propertys
+    f.addStep(clean.SetupPropertys())
+    return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py
index 7833233..c34d714 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -92,6 +92,8 @@ def gentoo_schedulers():
                                builderNames=["parse_build_log"])
     run_build_images_request = schedulers.Triggerable(name="run_build_images_request",
                                builderNames=["run_build_images_request"])
+    run_clean_db_request = schedulers.Triggerable(name="run_clean_db_request",
+                               builderNames=["run_clean_db_request"])
     s = []
     s.append(create_images)
     s.append(scheduler_update_db)
@@ -101,4 +103,5 @@ def gentoo_schedulers():
     s.append(build_request_data)
     s.append(run_build_request)
     s.append(parse_build_log)
+    s.append(run_clean_db_request)
     return s
diff --git a/buildbot_gentoo_ci/steps/clean.py b/buildbot_gentoo_ci/steps/clean.py
new file mode 100644
index 0000000..73733f1
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/clean.py
@@ -0,0 +1,97 @@
+# Copyright 2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
+from buildbot.plugins import steps, util
+
+class SetupPropertys(BuildStep):
+    name = 'Setup propertys for clean db'
+    description = 'Running'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        aftersteps_list = []
+        # Check on deleted ebuild on package
+        deleted_ebuilds_data = yield self.gentooci.db.versions.getEbuildsByPackage(self.getProperty("package_data")['uuid'], deleted=True)
+        # check if older then X days
+        #FIXME: set days in config and move the check to db side
+        days = 30
+        days_in_s = 86400 * days
+        now = int(self.master.reactor.seconds())
+        for ebuild_data in deleted_ebuilds_data:
+            no_bug = True
+            # check if older the days
+            if ebuild_data['deleted_at'] < (now - days_in_s):
+                print(f"Ebuild : {ebuild_data['uuid']} will be removed")
+                # get build_data for deleted_ebuild_data['uuid']
+                build_data_list = yield self.gentooci.db.builds.getBuildsByVersionUuid(ebuild_data['uuid'])
+                print(build_data_list)
+                for build_data in build_data_list:
+                    if build_data['bug_id'] == 0:
+                        # get buildsets
+                        print(f"Build : {build_data['id']} will be removed")
+                        yield self.build.addStepsAfterCurrentStep([RemoveBuildFromDb(build_data)])
+                    else:
+                        no_bug = False
+                if no_bug:
+                    yield self.build. addStepsAfterLastStep([RemoveEbuildFromDb(ebuild_data)])
+        return SUCCESS
+
+class RemoveEbuildFromDb(BuildStep):
+    name = 'Clean ebuild from db'
+    description = 'Running'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, ebuild_data, **kwargs):
+        self.ebuild_data = ebuild_data
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        yield self.gentooci.db.versions.removeVersionKeyword(self.ebuild_data['uuid'])
+        yield self.gentooci.db.versions.removeVersionMetadata(self.ebuild_data['uuid'])
+        yield self.gentooci.db.versions.removeVersion(self.ebuild_data['uuid'])
+        self.descriptionDone = f"Ebuild : {self.ebuild_data['uuid']} will be removed"
+        return SUCCESS
+
+class RemoveBuildFromDb(BuildStep):
+    name = 'Clean builds from db'
+    description = 'Running'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, build_data, **kwargs):
+        self.build_data = build_data
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        if self.build_data['buildbot_build_id'] != 0:
+            self.descriptionDone = f"BuildBot Build : {self.build_data['buildbot_build_id']} will be removed"
+            bb_build_data = yield self.master.db.builds.getBuild(self.build_data['buildbot_build_id'])
+            # remove buildbot data on db
+            # remove steps and logs
+            yield self.master.db.logs.deleteLogChunks(self.build_data['buildbot_build_id'])
+            # remove propertys
+            # get buildsets
+            # remove build_data
+        yield self.gentooci.db.builds.removeBuild(self.build_data['id'])
+        return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py
index 13eafa2..847ba12 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -12,6 +12,7 @@ from twisted.python import log
 from buildbot.process.buildstep import BuildStep
 from buildbot.process.results import SUCCESS
 from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
 from buildbot.process import remotecommand
 from buildbot.plugins import steps
 
@@ -282,3 +283,47 @@ class TriggerCheckForV(BuildStep):
             )
         yield self.build.addStepsAfterCurrentStep(addStepUpdateVData)
         return SUCCESS
+
+class TriggerCleanPackageDb(BuildStep):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    name = 'TriggerCleanPackageDb'
+    description = 'Running'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        aftersteps_list = []
+        # Check on deleted ebuild on package
+        deleted_ebuilds_data = yield self.gentooci.db.versions.getEbuildsByPackage(self.getProperty("package_data")['uuid'], deleted=True)
+        # check if older then X days
+        #FIXME: set days in config and move the check to db side
+        days = 30
+        days_in_s = 86400 * days
+        needcleaning  = 0
+        now = int(self.master.reactor.seconds())
+        for deleted_ebuild_data in deleted_ebuilds_data:
+            # check if older the days
+            if deleted_ebuild_data['deleted_at'] < (now - days_in_s):
+                needcleaning = needcleaning  + 1
+        print(f"{str(needcleaning)} ebuilds need cleaning in db")
+        if needcleaning > 0:
+            aftersteps_list.append(
+                steps.Trigger(
+                    schedulerNames=['run_clean_db_request'],
+                    waitForFinish=False,
+                    updateSourceStamp=False,
+                    set_properties={
+                        'package_data' : self.getProperty("package_data"),
+                    }
+                )
+            )
+            yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+            self.descriptionDone = f"Need DB Cleaning: Yes"
+            return SUCCESS
+        self.descriptionDone = f"Need DB Cleaning: No"
+        return SKIPPED