Add logic for importing devices from C3

README.md

@@ -51,8 +51,11 @@ curl http://0.0.0.0:8080/v1/certification/status -X POST -H "Content-Type: appli
 
 ### Load the data from C3
 
-This approach populates the DB with the data from C3 (staging instance by default). To build and run the
-container, execute the following command:
+This approach populates the DB with the data from C3 (staging instance by default).
+Keep in mind that importing data from staging or production takes some time, you probably
+can consider importing data from your local C3 instance with sample data.
+
+To build and run the container with staging data, execute the following command:
 
 ```bash
 docker-compose up --attach-dependencies --build hwapi-dev

server/.gitignore

@@ -178,4 +178,5 @@ pyrightconfig.json
 # End of https://www.toptal.com/developers/gitignore/api/python
 
 # SQLite3 db
-**/*.db
+*.db
+*.db-journal

server/docker-compose.yml

@@ -12,6 +12,7 @@ services:
     working_dir: /home/code
     volumes:
       - ./hwapi:/home/code/hwapi
+      - ./scripts:/home/code/scripts
     ports:
       - "8080:8080"
 

server/hwapi/data_models/enums.py

@@ -17,11 +17,103 @@
 # Written by:
 #        Nadzeya Hutsko <nadzeya.hutsko@canonical.com>
 
-
 from enum import Enum
 
 
 class CertificationStatus(str, Enum):
     CERTIFIED = "Certified"
     PARTIALLY_CERTIFIED = "Partially Certified"
     NOT_SEEN = "Not Seen"
+
+
+class BusType(str, Enum):
+    apex = "apex"
+    ata_device = "ata_device"
+    backlight = "backlight"
+    block = "block"
+    bluetooth = "bluetooth"
+    cciss = "cciss"
+    ccw = "ccw"
+    dmi = "dmi"
+    drm = "drm"
+    firewire = "firewire"
+    gameport = "gameport"
+    hid = "hid"
+    hidraw = "hidraw"
+    i2c = "i2c"
+    ide = "ide"
+    ieee80211 = "ieee80211"
+    infiniband = "infiniband"
+    input = "input"
+    memstick_host = "memstick_host"
+    misc = "misc"
+    mmc = "mmc"
+    mmc_host = "mmc_host"
+    mmc_rpmb = "mmc_rpmb"
+    mtd = "mtd"
+    nd = "nd"
+    net = "net"
+    nvme = "nvme"
+    pci = "pci"
+    platform = "platform"
+    pnp = "pnp"
+    power_supply = "power_supply"
+    ppdev = "ppdev"
+    rc = "rc"
+    rfkill = "rfkill"
+    scsi = "scsi"
+    scsi_host = "scsi_host"
+    sdio = "sdio"
+    serial = "serial"
+    serio = "serio"
+    sound = "sound"
+    tty = "tty"
+    usb = "usb"
+    usb_device = "usb_device"
+    vchiq = "vchiq"
+    video4linux = "video4linux"
+    virtio = "virtio"
+    virtual = "virtual"
+
+
+class DeviceCategory(str, Enum):
+    ACCELEROMETER = "ACCELEROMETER"
+    AUDIO = "AUDIO"
+    BIOS = "BIOS"
+    BLUETOOTH = "BLUETOOTH"
+    BMC_NETWORK = "BMC_NETWORK"
+    BOARD = "BOARD"
+    CANBUS = "CANBUS"
+    CAPTURE = "CAPTURE"
+    CARDREADER = "CARDREADER"
+    CDROM = "CDROM"
+    CHASSIS = "CHASSIS"
+    DISK = "DISK"
+    EFI = "EFI"
+    FIREWIRE = "FIREWIRE"
+    FLOPPY = "FLOPPY"
+    HIDRAW = "HIDRAW"
+    IDE = "IDE"
+    INFINIBAND = "INFINIBAND"
+    KEYBOARD = "KEYBOARD"
+    MMAL = "MMAL"
+    MODEM = "MODEM"
+    MOUSE = "MOUSE"
+    NETWORK = "NETWORK"
+    OTHER = "OTHER"
+    PRINTER = "PRINTER"
+    PROCESSOR = "PROCESSOR"
+    RAID = "RAID"
+    SCSI = "SCSI"
+    SOCKET = "SOCKET"
+    SOCKETCAN = "SOCKETCAN"
+    SYSTEM = "SYSTEM"
+    TOUCH = "TOUCH"
+    TOUCHPAD = "TOUCHPAD"
+    TOUCHSCREEN = "TOUCHSCREEN"
+    TPU = "TPU"
+    USB = "USB"
+    VIDEO = "VIDEO"
+    WATCHDOG = "WATCHDOG"
+    WIRELESS = "WIRELESS"
+    WWAN = "WWAN"

server/hwapi/data_models/models.py

@@ -20,10 +20,20 @@
 
 from datetime import date, datetime
 
-from sqlalchemy import ForeignKey, String, UniqueConstraint
+from sqlalchemy import (
+    ForeignKey,
+    String,
+    UniqueConstraint,
+    Table,
+    Column,
+    Integer,
+    Enum,
+)
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 from sqlalchemy.orm import relationship
 
+from .enums import BusType, DeviceCategory
+
 
 class Base(DeclarativeBase):
     """Base model for all the models"""
@@ -38,6 +48,7 @@ class Vendor(Base):
     # Relationships
     platforms: Mapped[list["Platform"]] = relationship(back_populates="vendor")
     bioses: Mapped[list["Bios"]] = relationship(back_populates="vendor")
+    devices: Mapped[list["Device"]] = relationship(back_populates="vendor")
 
 
 class Platform(Base):
@@ -66,7 +77,7 @@ class Configuration(Base):
 
 class Machine(Base):
     __tablename__ = "machine"
-    canonical_id: Mapped[str] = mapped_column(unique=True, nullable=False)
+    canonical_id: Mapped[str] = mapped_column(unique=True, nullable=False, index=True)
     # Relationships
     configuration_id: Mapped[int] = mapped_column(
         ForeignKey("configuration.id"), index=True
@@ -77,8 +88,10 @@ class Machine(Base):
 
 class Certificate(Base):
     __tablename__ = "certificate"
+    name: Mapped[str] = mapped_column(
+        String(80), unique=True, nullable=False, index=True
+    )
     created_at: Mapped[datetime] = mapped_column(nullable=False)
-    name: Mapped[str] = mapped_column(String(80), unique=True, nullable=True)
     completed: Mapped[datetime] = mapped_column(nullable=True)
     # Relationships
     machine_id: Mapped[int] = mapped_column(
@@ -126,6 +139,14 @@ class Bios(Base):
     reports: Mapped[list["Report"]] = relationship(back_populates="bios")
 
 
+device_report_association = Table(
+    "device_report_association",
+    Base.metadata,
+    Column("device_id", Integer, ForeignKey("device.id"), primary_key=True),
+    Column("report_id", Integer, ForeignKey("report.id"), primary_key=True),
+)
+
+
 class Report(Base):
     __tablename__ = "report"
     architecture: Mapped[str] = mapped_column(nullable=True)
@@ -142,3 +163,25 @@ class Report(Base):
         ForeignKey("certificate.id"), index=True, nullable=False
     )
     certificate: Mapped[Certificate] = relationship(back_populates="reports")
+    devices = relationship(
+        "Device", secondary=device_report_association, back_populates="reports"
+    )
+
+
+class Device(Base):
+    __tablename__ = "device"
+    identifier: Mapped[str] = mapped_column(nullable=False, index=True)
+    name: Mapped[str] = mapped_column(nullable=False, index=True)
+    subproduct_name: Mapped[str] = mapped_column(nullable=False)
+    device_type: Mapped[str] = mapped_column(nullable=False)
+    bus: Mapped[str] = mapped_column(Enum(BusType), nullable=False)
+    version: Mapped[str] = mapped_column(String(10), nullable=False)
+    subsystem: Mapped[str] = mapped_column(nullable=False)
+    category: Mapped[str] = mapped_column(Enum(DeviceCategory), nullable=False)
+    codename: Mapped[str] = mapped_column(String(40), nullable=False)
+    # Relationships
+    vendor_id: Mapped[int] = mapped_column(ForeignKey("vendor.id"), index=True)
+    vendor = relationship("Vendor", back_populates="devices")
+    reports = relationship(
+        "Report", secondary=device_report_association, back_populates="devices"
+    )

server/hwapi/data_models/repository.py

@@ -18,6 +18,7 @@
 #        Nadzeya Hutsko <nadzeya.hutsko@canonical.com>
 
 
+from typing import Type, Any
 from sqlalchemy.orm import Session
 
 from hwapi.data_models import models
@@ -76,11 +77,33 @@ def get_latest_certificate_for_configs(
     return latest_certificate
 
 
-def get_or_create(db: Session, model, **kwargs):
-    """Check if the object with the specified parameters exists. If not, create it"""
+def get_or_create(
+    db: Session,
+    model: Type[models.Base],
+    defaults: dict[str, Any] | None = None,
+    **kwargs
+) -> tuple:
+    """
+    Retrieves an object from the database based on the provided kwargs. If it doesn't
+    exist, it creates the object using both the kwargs and any additional default
+    attributes provided.
+
+    :param db: Database session
+    :param model: SQLAlchemy Model class
+    :param defaults: Dictionary of default values to initialise the model
+    :param kwargs: Keyword arguments corresponding to the model's attributes used for
+                   lookup and creation
+    :return: A tuple of (instance, created), where `created` is a boolean indicating
+             whether the instance was created in this call
+    """
     instance = db.query(model).filter_by(**kwargs).first()
-    if not instance:
-        instance = model(**kwargs)
-        db.add(instance)
-        db.commit()
-    return instance
+    if instance:
+        return instance, False
+
+    if defaults is not None:
+        kwargs.update(defaults)
+
+    instance = model(**kwargs)
+    db.add(instance)
+    db.commit()
+    return instance, True