From 41783fe24575a0465a56a144e1cbc3db49dd28f4 Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Tue, 2 May 2023 15:58:07 -0400 Subject: [PATCH 01/17] multiple fix circular import reference make CurrentUser schema allow_none=True for all fields --- .../api/json_api/account/current_user.py | 106 ++++++++++-------- .../api/json_api/tasks/task_filters.py | 12 +- .../api/openapi/openapi_spec.py | 5 +- axonius_api_client/api/system/instances.py | 2 +- axonius_api_client/api/system/settings.py | 2 +- .../cli/grp_assets/cmds_run_enforcement.py | 3 +- .../grp_enforcements/grp_tasks/export_get.py | 4 +- .../grp_tasks/export_get_filters.py | 7 +- .../grp_enforcements/grp_tasks/options_get.py | 8 +- .../grp_tasks/options_get_filters.py | 4 +- axonius_api_client/cli/helps.py | 28 +++-- axonius_api_client/cli/options.py | 5 +- axonius_api_client/constants/asset_helpers.py | 5 +- 13 files changed, 106 insertions(+), 85 deletions(-) diff --git a/axonius_api_client/api/json_api/account/current_user.py b/axonius_api_client/api/json_api/account/current_user.py index 65ce3c03..99eac6fe 100644 --- a/axonius_api_client/api/json_api/account/current_user.py +++ b/axonius_api_client/api/json_api/account/current_user.py @@ -11,32 +11,40 @@ class CurrentUserSchema(BaseSchemaJson): - """Schema for receiving current user response.""" - - id = mm_fields.Str() - uuid = mm_fields.Str() - role_id = mm_fields.Str() - role_name = mm_fields.Str() - user_name = mm_fields.Str() - password = mm_fields.Str() - source = mm_fields.Str() - first_name = mm_fields.Str() - last_name = mm_fields.Str() - email = mm_fields.Str() - department = mm_fields.Str() - title = mm_fields.Str() - pic_name = mm_fields.Str() - predefined = SchemaBool() - is_axonius_role = SchemaBool() - interests = mm_fields.Dict(load_default=dict, dump_default=dict) - permissions = mm_fields.Dict(load_default=dict, dump_default=dict) - data_scope = mm_fields.Dict(dump_default=dict, load_default=dict) + """Schema for receiving current user response. + + This schema does not match the REST API definition because it does not + follow its own rules. email is defined as allow_none=False in REST API, + however it can come back as None from REST API. + + Since it happened once, we are setting allow_none=True on all fields for + safety since we get this object anytime str(Connect()) is called. + """ + + id = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + uuid = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + role_id = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + role_name = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + user_name = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + password = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + source = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + first_name = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + last_name = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + email = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + department = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + title = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + pic_name = mm_fields.Str(allow_none=True, load_default=None, dump_default=None) + predefined = SchemaBool(allow_none=True, load_default=False, dump_default=False) + is_axonius_role = SchemaBool(allow_none=True, load_default=False, dump_default=False) + interests = mm_fields.Dict(allow_none=True, load_default=dict, dump_default=dict) + permissions = mm_fields.Dict(allow_none=True, load_default=dict, dump_default=dict) + data_scope = mm_fields.Dict(allow_none=True, dump_default=dict, load_default=dict) allowed_scopes_impersonation = mm_fields.List( - mm_fields.Str, load_default=list, dump_default=list + mm_fields.Str, allow_none=True, load_default=list, dump_default=list ) - timeout = mm_fields.Integer(load_default=None, dump_default=None, alllow_none=True) - last_updated = SchemaDatetime(load_default=None, dump_default=None, allow_none=True) - last_login = SchemaDatetime(load_default=None, dump_default=None, allow_none=True) + timeout = mm_fields.Integer(allow_none=True, load_default=None, dump_default=None) + last_updated = SchemaDatetime(allow_none=True, load_default=None, dump_default=None) + last_login = SchemaDatetime(allow_none=True, load_default=None, dump_default=None) class Meta: """JSONAPI config.""" @@ -56,24 +64,24 @@ def get_model_cls() -> t.Any: class CurrentUser(BaseModel): """Model for receiving current user response.""" - id: str = field_from_mm(SCHEMA, "id") - uuid: str = field_from_mm(SCHEMA, "uuid") - role_id: str = field_from_mm(SCHEMA, "role_id") - role_name: str = field_from_mm(SCHEMA, "role_name") - user_name: str = field_from_mm(SCHEMA, "user_name") - password: str = field_from_mm(SCHEMA, "password") - source: str = field_from_mm(SCHEMA, "source") - first_name: str = field_from_mm(SCHEMA, "first_name") - last_name: str = field_from_mm(SCHEMA, "last_name") - email: str = field_from_mm(SCHEMA, "email") - department: str = field_from_mm(SCHEMA, "department") - title: str = field_from_mm(SCHEMA, "title") - pic_name: str = field_from_mm(SCHEMA, "pic_name") - predefined: bool = field_from_mm(SCHEMA, "predefined") - is_axonius_role: bool = field_from_mm(SCHEMA, "is_axonius_role") - permissions: dict = field_from_mm(SCHEMA, "permissions", repr=False) - interests: dict = field_from_mm(SCHEMA, "interests") - data_scope: dict = field_from_mm(SCHEMA, "data_scope") + id: t.Optional[str] = field_from_mm(SCHEMA, "id") + uuid: t.Optional[str] = field_from_mm(SCHEMA, "uuid") + role_id: t.Optional[str] = field_from_mm(SCHEMA, "role_id") + role_name: t.Optional[str] = field_from_mm(SCHEMA, "role_name") + user_name: t.Optional[str] = field_from_mm(SCHEMA, "user_name") + password: t.Optional[str] = field_from_mm(SCHEMA, "password") + source: t.Optional[str] = field_from_mm(SCHEMA, "source") + first_name: t.Optional[str] = field_from_mm(SCHEMA, "first_name") + last_name: t.Optional[str] = field_from_mm(SCHEMA, "last_name") + email: t.Optional[str] = field_from_mm(SCHEMA, "email") + department: t.Optional[str] = field_from_mm(SCHEMA, "department") + title: t.Optional[str] = field_from_mm(SCHEMA, "title") + pic_name: t.Optional[str] = field_from_mm(SCHEMA, "pic_name") + predefined: t.Optional[bool] = field_from_mm(SCHEMA, "predefined") + is_axonius_role: t.Optional[bool] = field_from_mm(SCHEMA, "is_axonius_role") + permissions: t.Optional[dict] = field_from_mm(SCHEMA, "permissions", repr=False) + interests: t.Optional[dict] = field_from_mm(SCHEMA, "interests") + data_scope: t.Optional[dict] = field_from_mm(SCHEMA, "data_scope") allowed_scopes_impersonation: t.List[str] = field_from_mm( SCHEMA, "allowed_scopes_impersonation" ) @@ -91,14 +99,20 @@ def get_schema_cls() -> t.Any: return CurrentUserSchema @property - def last_login_seconds_ago(self) -> float: + def last_login_seconds_ago(self) -> t.Optional[float]: """Get the number of seconds since last login.""" - return round(self.last_login_delta.total_seconds(), 2) + delta = self.last_login_delta + if isinstance(delta, datetime.timedelta): + return round(delta.total_seconds(), 2) @property - def last_login_delta(self) -> datetime.timedelta: + def last_login_delta(self) -> t.Optional[datetime.timedelta]: """Get the timedelta since last login.""" - return datetime.datetime.now(datetime.timezone.utc) - self.last_login + from ....tools import dt_parse + + last_login = dt_parse(self.last_login, allow_none=True) + if last_login: + return datetime.datetime.now(datetime.timezone.utc) - last_login @property def str_connect(self) -> str: diff --git a/axonius_api_client/api/json_api/tasks/task_filters.py b/axonius_api_client/api/json_api/tasks/task_filters.py index cd68c6bb..e8e55ffa 100644 --- a/axonius_api_client/api/json_api/tasks/task_filters.py +++ b/axonius_api_client/api/json_api/tasks/task_filters.py @@ -7,12 +7,12 @@ import marshmallow_jsonapi.fields as mm_fields import click -from axonius_api_client.constants.api import RE_PREFIX -from axonius_api_client.constants.ctypes import PatternLike, TypeMatch -from axonius_api_client.constants.general import SPLITTER -from axonius_api_client.exceptions import NotFoundError -from axonius_api_client.parsers.matcher import Matcher -from axonius_api_client.tools import coerce_int, listify +from ....constants.api import RE_PREFIX +from ....constants.ctypes import PatternLike, TypeMatch +from ....constants.general import SPLITTER +from ....exceptions import NotFoundError +from ....parsers.matcher import Matcher +from ....tools import coerce_int, listify from ..base2 import BaseModel, BaseSchema from ..custom_fields import field_from_mm diff --git a/axonius_api_client/api/openapi/openapi_spec.py b/axonius_api_client/api/openapi/openapi_spec.py index 823dd299..7205b9be 100644 --- a/axonius_api_client/api/openapi/openapi_spec.py +++ b/axonius_api_client/api/openapi/openapi_spec.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- """API for working with the OpenAPI specification file.""" - -from axonius_api_client.api.api_endpoints import ApiEndpoints -from axonius_api_client.api.mixins import ModelMixins +from ..api_endpoints import ApiEndpoints +from ..mixins import ModelMixins class OpenAPISpec(ModelMixins): diff --git a/axonius_api_client/api/system/instances.py b/axonius_api_client/api/system/instances.py index 0cfbe1f8..859c5e15 100644 --- a/axonius_api_client/api/system/instances.py +++ b/axonius_api_client/api/system/instances.py @@ -564,7 +564,7 @@ def admin_script_upload_path( **kwargs: passed to :meth:`upload_script` """ if is_url(value=path): - from axonius_api_client.projects.url_parser import UrlParser + from ...projects.url_parser import UrlParser parser = UrlParser(url=path) path_part = pathlib.Path(parser.parsed.path) diff --git a/axonius_api_client/api/system/settings.py b/axonius_api_client/api/system/settings.py index 16f0c9d0..9930a364 100644 --- a/axonius_api_client/api/system/settings.py +++ b/axonius_api_client/api/system/settings.py @@ -3,7 +3,7 @@ import pathlib from typing import Any, List, Optional, Tuple, Union -from axonius_api_client.projects import cert_human +from ...projects import cert_human from ...constants.api import USE_CA_PATH from ...exceptions import ApiError, NotFoundError from ...parsers.config import config_build, config_unchanged, config_unknown, parse_settings diff --git a/axonius_api_client/cli/grp_assets/cmds_run_enforcement.py b/axonius_api_client/cli/grp_assets/cmds_run_enforcement.py index bcdb9675..649cc2f2 100644 --- a/axonius_api_client/cli/grp_assets/cmds_run_enforcement.py +++ b/axonius_api_client/cli/grp_assets/cmds_run_enforcement.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Command line interface for Axonius API Client.""" -from axonius_api_client.constants.fields import AXID - +from ...constants.fields import AXID from ..context import CONTEXT_SETTINGS, click from ..options import AUTH, add_options diff --git a/axonius_api_client/cli/grp_enforcements/grp_tasks/export_get.py b/axonius_api_client/cli/grp_enforcements/grp_tasks/export_get.py index d93bdd2c..ac90f17a 100644 --- a/axonius_api_client/cli/grp_enforcements/grp_tasks/export_get.py +++ b/axonius_api_client/cli/grp_enforcements/grp_tasks/export_get.py @@ -4,8 +4,8 @@ import csv import io -from axonius_api_client.api.json_api.tasks import Task -from axonius_api_client.tools import json_dump, path_write +from ....api.json_api.tasks import Task +from ....tools import json_dump, path_write from ...context import click diff --git a/axonius_api_client/cli/grp_enforcements/grp_tasks/export_get_filters.py b/axonius_api_client/cli/grp_enforcements/grp_tasks/export_get_filters.py index cd2b38b4..1f713284 100644 --- a/axonius_api_client/cli/grp_enforcements/grp_tasks/export_get_filters.py +++ b/axonius_api_client/cli/grp_enforcements/grp_tasks/export_get_filters.py @@ -1,10 +1,7 @@ # -*- coding: utf-8 -*- """Command line interface for Axonius API Client.""" -from axonius_api_client.api.json_api.tasks.task_filters import ( - TaskFilters, - ATTR_MAP, -) -from axonius_api_client.tools import json_dump, path_write +from ....api.json_api.tasks.task_filters import TaskFilters, ATTR_MAP +from ....tools import json_dump, path_write from ...context import click diff --git a/axonius_api_client/cli/grp_enforcements/grp_tasks/options_get.py b/axonius_api_client/cli/grp_enforcements/grp_tasks/options_get.py index 4f2e0b75..6d55c943 100644 --- a/axonius_api_client/cli/grp_enforcements/grp_tasks/options_get.py +++ b/axonius_api_client/cli/grp_enforcements/grp_tasks/options_get.py @@ -2,10 +2,10 @@ """Common options for the "enforcements tasks" group of commands.""" import click -from axonius_api_client.api.json_api.count_operator import OperatorTypes -from axonius_api_client.api.json_api.paging_state import PagingState -from axonius_api_client.constants.api import RE_PREFIX -from axonius_api_client.constants.general import SPLITTER +from ....api.json_api.count_operator import OperatorTypes +from ....api.json_api.paging_state import PagingState +from ....constants.api import RE_PREFIX +from ....constants.general import SPLITTER from .export_get import DEFAULT_EXPORT_FORMAT, EXPORT_FORMATS from ...options import AUTH, OPT_EXPORT_FILE, OPT_EXPORT_OVERWRITE diff --git a/axonius_api_client/cli/grp_enforcements/grp_tasks/options_get_filters.py b/axonius_api_client/cli/grp_enforcements/grp_tasks/options_get_filters.py index bfd47e2a..4b883f88 100644 --- a/axonius_api_client/cli/grp_enforcements/grp_tasks/options_get_filters.py +++ b/axonius_api_client/cli/grp_enforcements/grp_tasks/options_get_filters.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- """Command line interface for Axonius API Client.""" -from axonius_api_client.api.json_api.tasks.task_filters import ( - build_include_options, -) +from ....api.json_api.tasks.task_filters import build_include_options from ...context import click from ...options import AUTH, OPT_EXPORT_FILE, OPT_EXPORT_OVERWRITE from .export_get_filters import EXPORT_FORMATS, DEFAULT_EXPORT_FORMAT diff --git a/axonius_api_client/cli/helps.py b/axonius_api_client/cli/helps.py index e9a8b06f..f8053c50 100644 --- a/axonius_api_client/cli/helps.py +++ b/axonius_api_client/cli/helps.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Command line interface for Axonius API Client.""" from ..constants.wizards import Docs -from ..constants.asset_helpers import ASSETS_HELPERS HELPSTR_AUTH = """ Detailed help for authentication: @@ -219,12 +218,21 @@ - node key now needs to be 'node_name' """ -HELPSTRS = {} -HELPSTRS["auth"] = HELPSTR_AUTH -HELPSTRS["assetexport"] = HELPSTR_EXPORT_ASSET -HELPSTRS["selectfields"] = HELPSTR_SELECT_FIELDS -HELPSTRS["query"] = HELPSTR_QUERY -HELPSTRS["wizard"] = Docs.TEXT -HELPSTRS["wizard_csv"] = Docs.CSV -HELPSTRS["multiple_cnx_json"] = HELPSTR_MULTI_CNX_JSON -HELPSTRS["asset_helper"] = ASSETS_HELPERS.to_str() + +def asset_helper(**kwargs) -> str: + """Return the help string for the asset helpers""" + from ..constants.asset_helpers import ASSETS_HELPERS + + return ASSETS_HELPERS.to_str() + + +HELPSTRS = { + "auth": HELPSTR_AUTH, + "assetexport": HELPSTR_EXPORT_ASSET, + "selectfields": HELPSTR_SELECT_FIELDS, + "query": HELPSTR_QUERY, + "wizard": Docs.TEXT, + "wizard_csv": Docs.CSV, + "multiple_cnx_json": HELPSTR_MULTI_CNX_JSON, + "asset_helper": asset_helper, +} diff --git a/axonius_api_client/cli/options.py b/axonius_api_client/cli/options.py index 8da31c0b..eada53b3 100644 --- a/axonius_api_client/cli/options.py +++ b/axonius_api_client/cli/options.py @@ -7,7 +7,6 @@ from ..constants.api import MAX_PAGE_SIZE, TABLE_FORMAT from ..tools import coerce_int from . import context -from .helps import HELPSTRS def build_filter_opt(value_type: str): @@ -46,7 +45,11 @@ def int_callback(ctx, param, value): def help_callback(ctx, param, value): """Pass.""" if value: + from .helps import HELPSTRS + helpstr = HELPSTRS[value] + if callable(helpstr): + helpstr = helpstr(ctx=ctx, param=param, value=value) click.secho(helpstr, err=True, fg="blue") ctx.exit(0) diff --git a/axonius_api_client/constants/asset_helpers.py b/axonius_api_client/constants/asset_helpers.py index e3d21b1d..f2601071 100644 --- a/axonius_api_client/constants/asset_helpers.py +++ b/axonius_api_client/constants/asset_helpers.py @@ -3,7 +3,6 @@ import typing as t from ..data import BaseData -from ..tools import json_dump, listify def to_json_api(value: t.Any, schema: str) -> dict: @@ -36,6 +35,8 @@ class AssetsHelper(BaseData): @staticmethod def join_path(values: t.List[str]) -> str: """Join a list of strings with ' => '""" + from axonius_api_client.tools import listify + return " => ".join(listify(values)) @property @@ -61,6 +62,8 @@ def to_str_short(self) -> str: def to_str(self) -> str: """Return a string describing this helper.""" + from axonius_api_client.tools import json_dump + value: str = f""" ## {self.name} From c4a26839640d1d443cbf749647edd4db4154b174 Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Tue, 2 May 2023 17:40:48 -0400 Subject: [PATCH 02/17] adding column filters to create saved query numerous lint fixes and docstring updates too get rid of credentials test in LoginRequest, it breaks AuthNull fix signup usage in CLI scripts --- axonius_api_client/api/assets/asset_mixin.py | 2 +- axonius_api_client/api/assets/saved_query.py | 203 +++++++++++++----- .../api/json_api/account/login_request.py | 27 --- axonius_api_client/cli/context.py | 4 +- .../cli/grp_tools/cmd_signup.py | 8 +- .../cli/grp_tools/cmd_system_status.py | 28 +-- .../grp_tools/cmd_use_token_reset_token.py | 5 +- axonius_api_client/tests/conftest.py | 10 +- 8 files changed, 169 insertions(+), 118 deletions(-) diff --git a/axonius_api_client/api/assets/asset_mixin.py b/axonius_api_client/api/assets/asset_mixin.py index d063281c..31a86c20 100644 --- a/axonius_api_client/api/assets/asset_mixin.py +++ b/axonius_api_client/api/assets/asset_mixin.py @@ -1153,6 +1153,7 @@ def get_generator( fields_default: include the default fields in :attr:`fields_default` fields_root: include all fields of an adapter that are not complex sub-fields fields_error: throw validation errors on supplied fields + fields_parsed: previously parsed fields max_rows: only return N rows max_pages: only return N pages row_start: start at row N @@ -1172,7 +1173,6 @@ def get_generator( wiz_entries: wizard expressions to create query from file_date: string to use in filename templates for {DATE} wiz_parsed: parsed output from a query wizard - fields_parsed: previously parsed fields sort_field_parsed: previously parsed sort field history_date_parsed: previously parsed history date initial_count: previously fetched initial count diff --git a/axonius_api_client/api/assets/saved_query.py b/axonius_api_client/api/assets/saved_query.py index a2c29ff0..d84e88bc 100644 --- a/axonius_api_client/api/assets/saved_query.py +++ b/axonius_api_client/api/assets/saved_query.py @@ -266,6 +266,7 @@ def update_fields( fields_fuzzy (t.Optional[t.Union[t.List[str], str]], optional): fields via fuzzy fields_default (bool, optional): Include default fields fields_root (t.Optional[str], optional): fields via root + fields_regex_root_only (bool, optional): only match root fields in fields_regex remove (bool, optional): remove supplied fields from saved query fields append (bool, optional): append supplied fields in value to pre-existing saved query fields @@ -294,6 +295,7 @@ def update_fields( sq.fields = value return self._update_handler(sq=sq, as_dataclass=as_dataclass) + # noinspection PyUnusedLocal def update_query( self, sq: MULTI, @@ -392,6 +394,10 @@ def copy( private (bool, optional): Set new sq as private asset_scope (bool, optional): Set new sq as asset scope query as_dataclass (bool, optional): Return saved query dataclass instead of dict + always_cached (bool, optional): Set new sq as always cached + folder (t.Optional[t.Union[str, FolderModel]], optional): Folder to create new sq in + create (bool, optional): Create folder if it doesn't exist + echo (bool, optional): Echo API response Returns: t.Union[dict, models.SavedQuery]: saved query dataclass or dict @@ -421,6 +427,8 @@ def get_by_multi( Args: sq (MULTI): str with name or uuid, or saved query dict or dataclass as_dataclass (bool, optional): Return saved query dataclass instead of dict + asset_scopes (bool, optional): Only search asset scope queries + cache (bool, optional): Get cached results **kwargs: passed to :meth:`get` Returns: @@ -470,12 +478,17 @@ def get_by_name( Examples: Get a saved query by name - >>> sq = apiobj.saved_query.get_by_name(name="test") - >>> sq['tags'] + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities + >>> data = apiobj.saved_query.get_by_name(name="test") + >>> data['tags'] ['Unmanaged Devices'] - >>> sq['description'][:80] + >>> data['description'][:80] 'Devices that have been seen by at least one agent or at least one endpoint manag' - >>> sq['view']['fields'] + >>> data['view']['fields'] [ 'adapters', 'specific_data.data.name', @@ -487,7 +500,7 @@ def get_by_name( 'specific_data.data.os.type', 'labels' ] - >>> sq['view']['query']['filter'][:80] + >>> data['view']['query']['filter'][:80] '(specific_data.data.adapter_properties == "Agent") or (specific_data.data.adapte' Args: @@ -517,7 +530,12 @@ def get_by_uuid( Examples: Get a saved query by uuid - >>> sq = apiobj.saved_query.get_by_uuid(value="5f76721ce4557d5cba93f59e") + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities + >>> data = apiobj.saved_query.get_by_uuid(value="5f76721ce4557d5cba93f59e") Args: value (str): uuid of saved query @@ -544,15 +562,18 @@ def get_by_tags( Examples: Get all saved queries with tagged with 'AD' - - >>> sqs = apiobj.saved_query.get_by_tags('AD') - >>> len(sqs) + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities + >>> data = apiobj.saved_query.get_by_tags('AD') + >>> len(data) 2 Get all saved queries with tagged with 'AD' or 'AWS' - - >>> sqs = apiobj.saved_query.get_by_tags(['AD', 'AWS']) - >>> len(sqs) + >>> data = apiobj.saved_query.get_by_tags(['AD', 'AWS']) + >>> len(data) 5 Args: @@ -578,7 +599,7 @@ def get_by_tags( found.append(sq) if not found: - raise SavedQueryTagsNotFoundError(value=value, valid=valid) + raise SavedQueryTagsNotFoundError(value=value, valid=list(valid)) return found if as_dataclass else [x.to_dict() for x in found] def get_tags_slow(self) -> t.List[str]: @@ -587,8 +608,13 @@ def get_tags_slow(self) -> t.List[str]: Examples: Get all known tags for all saved queries - >>> tags = apiobj.saved_query.get_tags() - >>> len(tags) + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities + >>> data = apiobj.saved_query.get_tags() + >>> len(data) 19 Returns: @@ -630,6 +656,7 @@ def get_query_history( gen = self.get_query_history_generator(**kwargs) return gen if generator else list(gen) + # noinspection PyShadowingBuiltins def get_query_history_generator( self, run_by: t.Optional[PatternLikeListy] = None, @@ -700,6 +727,7 @@ def get_query_history_generator( values=tags, enum_callback=self.get_tags, ) + # noinspection PyUnresolvedReferences request_obj.set_list( prop="modules", values=modules or self.parent.ASSET_TYPE, @@ -743,8 +771,13 @@ def get( Examples: Get all saved queries - >>> sqs = apiobj.saved_query.get() - >>> len(sqs) + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities + >>> data = apiobj.saved_query.get() + >>> len(data) 39 Args: @@ -776,13 +809,15 @@ def get_cached(self, **kwargs) -> t.List[t.Union[dict, models.SavedQuery]]: Examples: Get all saved queries - >>> sqs = apiobj.saved_query.get() - >>> len(sqs) + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities + >>> data = apiobj.saved_query.get() + >>> len(data) 39 - Args: - generator: return an iterator - Yields: t.Generator[QueryHistory, None, None]: if generator = True, saved query dataclass or dict @@ -794,6 +829,7 @@ def get_cached(self, **kwargs) -> t.List[t.Union[dict, models.SavedQuery]]: """ return list(self.get_generator(**kwargs)) + # noinspection PyProtectedMember def get_cached_single(self, value: t.Union[str, dict, models.SavedQuery]) -> models.SavedQuery: """Pass.""" name = models.SavedQuery._get_attr_value(value=value, attr="name") @@ -805,6 +841,7 @@ def get_cached_single(self, value: t.Union[str, dict, models.SavedQuery]) -> mod raise SavedQueryNotFoundError(sqs=items, details=f"name={name!r} and uuid={value!r}") + # noinspection PyUnresolvedReferences @property def query_by_asset_type(self) -> str: """Pass.""" @@ -839,11 +876,23 @@ def get_generator( log_level: t.Union[int, str] = LOG_LEVEL_API, query: t.Optional[str] = None, request_obj: t.Optional[models.SavedQueryGet] = None, - ) -> t.Generator[models.QueryHistory, None, None]: + ) -> t.Generator[models.SavedQuery, None, None]: """Get Saved Queries using a generator. Args: as_dataclass (bool, optional): Return saved query dataclass instead of dict + folder_id (str, optional): folder id, will return all if "all", otherwise + will return only saved queries directly in or under the folder + include_usage (bool, optional): include usage data + get_view_data (bool, optional): include view data + page_sleep (int, optional): sleep in seconds between pages + page_size (int, optional): page size + row_start (int, optional): row start + row_stop (int, optional): row stop + add_query_by_asset_type (bool, optional): add query by asset type to query string + log_level (int, optional): log level + query (str, optional): query to filter saved queries + request_obj (t.Optional[models.SavedQueryGet], optional): request object Yields: t.Generator[QueryHistory, None, None]: saved query dataclass or dict @@ -879,6 +928,11 @@ def add(self, as_dataclass: bool = AS_DATACLASS, **kwargs) -> t.Union[dict, mode Examples: Create a saved query using a :obj:`axonius_api_client.api.wizards.wizard.Wizard` + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities >>> parsed = apiobj.wizard_text.parse(content="simple hostname contains blah") >>> query = parsed["query"] >>> expressions = parsed["expressions"] @@ -887,7 +941,7 @@ def add(self, as_dataclass: bool = AS_DATACLASS, **kwargs) -> t.Union[dict, mode ... query=query, ... expressions=expressions, ... description="meep meep", - ... tags=["nyuck1", "nyuck2", "nyuck3"], + ... tags=["tag1", "tag2", "tag3"], ... ) Notes: @@ -923,9 +977,14 @@ def build_add_model( fields_fuzzy: t.Optional[t.Union[t.List[str], str]] = None, fields_default: bool = True, fields_root: t.Optional[str] = None, + fields_parsed: t.Optional[t.Union[dict, t.List[str]]] = None, sort_field: t.Optional[str] = None, sort_descending: bool = True, - column_filters: t.Optional[dict] = None, + sort_field_parsed: t.Optional[str] = None, + field_filters: t.Optional[t.List[dict]] = None, + excluded_adapters: t.Optional[t.List[dict]] = None, + asset_excluded_adapters: t.Optional[t.List[dict]] = None, + asset_filters: t.Optional[t.List[dict]] = None, gui_page_size: t.Optional[int] = None, private: bool = False, always_cached: bool = False, @@ -968,6 +1027,8 @@ def build_add_model( Args: name: name of saved query + description: description of saved query + query: query built by GUI or API query wizard wiz_entries (t.Optional[t.Union[str, t.List[dict]]]): API query wizard entries to parse into query and GUI query wizard expressions @@ -978,14 +1039,25 @@ def build_add_model( fields_regex: regex of fields to return for each asset fields_fuzzy: string to fuzzy match of fields to return for each asset fields_default: include the default fields defined in the parent asset object + fields_regex_root_only: only match fields in fields_regex that are not sub-fields of + other fields fields_root: include all fields of an adapter that are not complex sub-fields + fields_parsed: previously parsed fields sort_field: sort the returned assets on a given field sort_descending: reverse the sort of the returned assets - column_filters: NOT_SUPPORTED + field_filters: field filters to apply to this query + excluded_adapters: adapters to exclude from this query + asset_excluded_adapters: adapters to exclude from this query + asset_filters: asset filters to apply to this query gui_page_size: show N rows per page in GUI private: make this saved query private to current user always_cached: always keep this query cached asset_scope: make this query an asset scope query + folder: folder to create saved query in + create: create folder if it does not exist + echo: echo folder actions to stdout/stderr + sort_field_parsed: previously parsed sort field + Returns: models.SavedQueryCreate: saved query dataclass to create @@ -1000,7 +1072,7 @@ def build_add_model( root: FoldersModel = self.folders.get() fallback: t.Optional[FolderModel] = None if asset_scope: - self.auth.http.CLIENT.data_scopes.check_feature_enabled() + self.auth.CLIENT.data_scopes.check_feature_enabled() fallback: t.Optional[FolderModel] = root.path_asset_scope reason: str = f"Create Saved Query {name!r}" @@ -1022,37 +1094,43 @@ def build_add_model( gui_page_size = check_gui_page_size(size=gui_page_size) - fields = self.parent.fields.validate( - fields=fields, - fields_manual=fields_manual, - fields_regex=fields_regex, - fields_default=fields_default, - fields_root=fields_root, - fields_fuzzy=fields_fuzzy, - fields_regex_root_only=fields_regex_root_only, - fields_error=True, - ) - - if sort_field: - sort_field = self.parent.fields.get_field_name(value=sort_field) - - view = {} - view["query"] = {} - view["query"]["filter"] = query or "" - view["query"]["expressions"] = expressions or [] - view["query"]["search"] = None # TBD - view["query"]["meta"] = {} # TBD - view["query"]["meta"]["enforcementFilter"] = None # TBD - view["query"]["meta"]["uniqueAdapters"] = False # TBD - - if query_expr: - view["query"]["onlyExpressionsFilter"] = query_expr - - view["sort"] = {} - view["sort"]["desc"] = sort_descending - view["sort"]["field"] = sort_field or "" - view["fields"] = fields - view["pageSize"] = gui_page_size + if not isinstance(fields_parsed, (list, tuple)): + fields_parsed = self.parent.fields.validate( + fields=fields, + fields_manual=fields_manual, + fields_regex=fields_regex, + fields_default=fields_default, + fields_root=fields_root, + fields_fuzzy=fields_fuzzy, + fields_regex_root_only=fields_regex_root_only, + fields_error=True, + ) + if not isinstance(sort_field_parsed, str): + sort_field_parsed: str = self.parent.fields.get_field_name(value=sort_field) or "" + + view_sort: dict = { + "desc": sort_descending, + "field": sort_field_parsed, + } + view_query: dict = { + "filter": query or "", + "expressions": expressions or [], + "search": None, + "meta": {}, + "enforcementFilter": None, + "uniqueAdapters": False, + "onlyExpressionsFilter": query_expr or "", + } + view: dict = { + "fields": fields_parsed, + "pageSize": gui_page_size, + "sort": view_sort, + "query": view_query, + "colFilters": listify(field_filters), + "colExcludeAdapters": listify(excluded_adapters), + "assetConditionExpressions": listify(asset_filters), + "assetExcludeAdapters": listify(asset_excluded_adapters), + } return models.SavedQueryCreate.new_from_kwargs( name=name, description=description, @@ -1072,6 +1150,11 @@ def delete_by_name( Examples: Delete the saved query by name + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities >>> deleted = apiobj.saved_query.delete_by_name(name="test") Args: @@ -1164,7 +1247,7 @@ def _update_handler( t.Union[dict, models.SavedQuery]: saved query dataclass or dict """ ret = self._update_from_dataclass(obj=sq) - return self.get_by_multi(sq=ret, as_dataclass=True) + return self.get_by_multi(sq=ret, as_dataclass=as_dataclass) def _update_from_dataclass( self, obj: models.SavedQueryMixins, uuid: t.Optional[str] = None @@ -1192,6 +1275,7 @@ def _update_from_dataclass( self.get_cached.cache_clear() return response + # noinspection PyUnresolvedReferences def _add_from_dataclass(self, obj: models.SavedQueryCreate) -> models.SavedQuery: """Direct API method to create a saved query. @@ -1267,6 +1351,7 @@ def _get_query_history_run_from(self) -> ListValueSchema: api_endpoint = ApiEndpoints.saved_queries.get_run_from return api_endpoint.perform_request(http=self.auth.http) + # noinspection PyUnresolvedReferences def _get_tags(self) -> ListValueSchema: """Get the valid tags.""" api_endpoint = ApiEndpoints.saved_queries.get_tags diff --git a/axonius_api_client/api/json_api/account/login_request.py b/axonius_api_client/api/json_api/account/login_request.py index 04153ae0..ca3c20fb 100644 --- a/axonius_api_client/api/json_api/account/login_request.py +++ b/axonius_api_client/api/json_api/account/login_request.py @@ -5,7 +5,6 @@ from marshmallow_jsonapi import fields as mm_fields -from ....exceptions import AuthError from ..base import BaseModel, BaseSchemaJson from ..custom_fields import SchemaBool, field_from_mm @@ -68,33 +67,7 @@ class LoginRequest(BaseModel): SCHEMA: t.ClassVar[BaseSchemaJson] = SCHEMA - def __post_init__(self): - """Post init.""" - self.check_credentials() - @staticmethod def get_schema_cls() -> t.Any: """Pass.""" return LoginRequestSchema - - def _check_credential(self, attr: str) -> str: - """Check that a credential is a non-empty string.""" - value: t.Any = getattr(self, attr) - - if isinstance(value, str) and value.strip(): - value = value.strip() - setattr(self, attr, value) - return value - - field: mm_fields.Field = SCHEMA.declared_fields[attr] - description: str = field.metadata.get("description", f"{attr}") - msgs: t.List[str] = [ - f"Value provided for {description} is not a non-empty string", - f"Provided type {type(value)}, value: {value!r}", - ] - raise AuthError(msgs) - - def check_credentials(self): - """Check that username and password are not empty.""" - self._check_credential(attr="user_name") - self._check_credential(attr="password") diff --git a/axonius_api_client/cli/context.py b/axonius_api_client/cli/context.py index d95daa38..84d191b1 100644 --- a/axonius_api_client/cli/context.py +++ b/axonius_api_client/cli/context.py @@ -259,7 +259,9 @@ def wraperror(self): """Pass.""" return self._connect_args.get("wraperror", True) - def create_client(self, url, key, secret, **kwargs): + def create_client( + self, url: str, key: t.Optional[str] = None, secret: t.Optional[str] = None, **kwargs + ): """Pass.""" connect_args = {} connect_args.update(self._connect_args) diff --git a/axonius_api_client/cli/grp_tools/cmd_signup.py b/axonius_api_client/cli/grp_tools/cmd_signup.py index 69874642..675bf280 100644 --- a/axonius_api_client/cli/grp_tools/cmd_signup.py +++ b/axonius_api_client/cli/grp_tools/cmd_signup.py @@ -2,7 +2,6 @@ """Command line interface for Axonius API Client.""" import click -from ...api import Signup from ..options import add_options from .grp_common import EXPORT_FORMATS from .grp_options import OPT_ENV, OPT_EXPORT @@ -60,12 +59,11 @@ @click.pass_context def cmd(ctx, url, password, company_name, contact_email, export_format, env): """Perform the initial signup to an instance.""" - entry = Signup(url=url) + client = ctx.obj.create_client(url=url) with ctx.obj.exc_wrap(wraperror=ctx.obj.wraperror): - data = entry.signup( + data = client.signup.signup( password=password, company_name=company_name, contact_email=contact_email ) - - click.secho(EXPORT_FORMATS[export_format](data=data, signup=True, env=env, url=entry.http.url)) + click.secho(EXPORT_FORMATS[export_format](data=data, signup=True, env=env, url=client.http.url)) ctx.obj.echo_ok("Signup completed successfully!") ctx.exit(0) diff --git a/axonius_api_client/cli/grp_tools/cmd_system_status.py b/axonius_api_client/cli/grp_tools/cmd_system_status.py index d57b8a9c..9eb52391 100644 --- a/axonius_api_client/cli/grp_tools/cmd_system_status.py +++ b/axonius_api_client/cli/grp_tools/cmd_system_status.py @@ -5,7 +5,6 @@ import click -from ...api import Signup from ...tools import dt_now from ..context import CONTEXT_SETTINGS from ..options import URL, add_options @@ -52,26 +51,27 @@ def cmd(ctx, url, wait, sleep, max_wait): def get_status(): """Get the system status safely.""" try: - data = entry.system_status - message = data.msg - status_code = data.status_code - is_ready = data.is_ready + data = client.signup.system_status + _message = data.msg + _status_code = data.status_code + _is_ready = data.is_ready except Exception as exc: - message = f"HTTP Error: {exc}" - status_code = 1000 - is_ready = False + _message = f"HTTP Error: {exc}" + _status_code = 1000 + _is_ready = False msg = [ - f"URL: {entry.http.url}", + f"URL: {client.signup.http.url}", f"Date: {dt_now()}", - f"Message: {message}", - f"Status Code: {status_code}", - f"Ready: {is_ready}", + f"Message: {_message}", + f"Status Code: {_status_code}", + f"Ready: {_is_ready}", ] click.secho("\n".join(msg)) - return is_ready, status_code + return _is_ready, _status_code + + client = ctx.obj.create_client(url=url) - entry = Signup(url=url) with ctx.obj.exc_wrap(wraperror=ctx.obj.wraperror): is_ready, status_code = get_status() diff --git a/axonius_api_client/cli/grp_tools/cmd_use_token_reset_token.py b/axonius_api_client/cli/grp_tools/cmd_use_token_reset_token.py index adb43ffd..7e12aa77 100644 --- a/axonius_api_client/cli/grp_tools/cmd_use_token_reset_token.py +++ b/axonius_api_client/cli/grp_tools/cmd_use_token_reset_token.py @@ -2,7 +2,6 @@ """Command line interface for Axonius API Client.""" import click -from ...api import Signup from ..context import CONTEXT_SETTINGS from ..options import URL, add_options @@ -36,8 +35,8 @@ @click.pass_context def cmd(ctx, url, token, password): """Use a password reset token.""" - client = Signup(url=url) + client = ctx.obj.create_client(url=url) with ctx.obj.exc_wrap(wraperror=ctx.obj.wraperror): - name = client.use_password_reset_token(token=token, password=password) + name = client.signup.use_password_reset_token(token=token, password=password) ctx.obj.echo_ok(f"Password successfully reset for user {name!r}") diff --git a/axonius_api_client/tests/conftest.py b/axonius_api_client/tests/conftest.py index 9de33c7b..652b7191 100644 --- a/axonius_api_client/tests/conftest.py +++ b/axonius_api_client/tests/conftest.py @@ -15,7 +15,6 @@ get_arg_key, get_arg_secret, get_arg_url, - get_http, get_connect, get_arg_cf_token, get_arg_cf_error, @@ -255,13 +254,8 @@ def api_settings_ip(api_client): @pytest.fixture(scope="session") def api_signup(request): """Test utility.""" - from axonius_api_client.api import Signup - from axonius_api_client.auth import AuthNull - - http = get_http(request) - auth = AuthNull(http=http) - obj = Signup(auth=auth) - return obj + client = get_connect(request=request) + return client.signup @pytest.fixture(scope="session") From 9aed025773194781c9dad0208c81f1a9c2b012c7 Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Tue, 2 May 2023 20:29:56 -0400 Subject: [PATCH 03/17] strip bug hunting (prior commit: circular import fixes) --- .../adapters/fetch_history_response.py | 4 +++- .../api/json_api/saved_queries.py | 9 +++++--- axonius_api_client/parsers/wizards.py | 9 +++++--- axonius_api_client/tools.py | 23 ++++++++++--------- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/axonius_api_client/api/json_api/adapters/fetch_history_response.py b/axonius_api_client/api/json_api/adapters/fetch_history_response.py index d11482d0..d0f8aea2 100644 --- a/axonius_api_client/api/json_api/adapters/fetch_history_response.py +++ b/axonius_api_client/api/json_api/adapters/fetch_history_response.py @@ -211,7 +211,9 @@ def to_tablize(self) -> dict: def getval(prop: str, width: t.Optional[int] = 30) -> str: """Pass.""" value = getattr(self, prop, None) - if isinstance(width, int) and len(str(value)) > width: + if not isinstance(value, str): + value = "" if value is None else str(value) + if isinstance(width, int) and len(value) > width: value = textwrap.fill(value, width=width) prop = prop.replace("_", " ").title() return f"{prop}: {value}" diff --git a/axonius_api_client/api/json_api/saved_queries.py b/axonius_api_client/api/json_api/saved_queries.py index 40425383..b93e2215 100644 --- a/axonius_api_client/api/json_api/saved_queries.py +++ b/axonius_api_client/api/json_api/saved_queries.py @@ -711,7 +711,8 @@ def to_strs(self) -> t.List[str]: def to_tablize(self) -> dict: """Get tablize-able repr of this obj.""" col_info = [ - f"{k.upper()}={textwrap.fill(v or '', width=30)}" for k, v in self.col_info.items() + f"{k.upper()}={textwrap.fill('' if v is None else str(v), width=30)}" + for k, v in self.col_info.items() ] col_details = [f"{k}: {v}" for k, v in self.col_details.items()] ret = {} @@ -1066,7 +1067,7 @@ def to_csv(self) -> dict: def getval(prop): value = getattr(self, prop, None) if isinstance(value, list): - value = "\n".join(value) + value = "\n".join([str(x) for x in value]) return value return {k: getval(k) for k in self._props_csv()} @@ -1076,7 +1077,9 @@ def to_tablize(self) -> dict: def getval(prop, width=30): value = getattr(self, prop, None) - if isinstance(width, int) and len(str(value)) > width: + if not isinstance(value, str): + value = "" if value is None else str(value) + if isinstance(width, int) and len(value) > width: value = textwrap.fill(value, width=width) prop = prop.replace("_", " ").title() return f"{prop}: {value}" diff --git a/axonius_api_client/parsers/wizards.py b/axonius_api_client/parsers/wizards.py index 246de43b..b7787aeb 100644 --- a/axonius_api_client/parsers/wizards.py +++ b/axonius_api_client/parsers/wizards.py @@ -18,7 +18,7 @@ parse_ip_network, strip_right, ) -from .tables import tablize, tablize_sqs +from .tables import tablize CACHE_MAXSIZE: int = 4096 CACHE_TTL: int = 30 @@ -411,7 +411,9 @@ def check_enum( def enum_cb_sq(self, value: Any) -> str: """Pass.""" - data = self.get_sqs() + from ..api.json_api.saved_queries import SavedQuery + + data: List[SavedQuery] = self.get_sqs() value_check = lowish(value) for item in data: @@ -419,7 +421,8 @@ def enum_cb_sq(self, value: Any) -> str: return item.uuid err = f"No Saved Query found with name or UUID of {value!r} out of {len(data)} items" - err_table = tablize_sqs(data=data, err=err) + err_table = tablize(value=[x.to_tablize() for x in data], err=err) + # err_table = tablize_sqs(data=data, err=err) raise WizardError(err_table) def enum_cb_cnx_label(self, value: Any) -> str: diff --git a/axonius_api_client/tools.py b/axonius_api_client/tools.py index ad66735f..2a7ba878 100644 --- a/axonius_api_client/tools.py +++ b/axonius_api_client/tools.py @@ -33,8 +33,8 @@ PatternLike, PatternLikeListy, TypeDate, - TypeFloat, TypeDelta, + TypeFloat, ) from .constants.general import ( DAYS_MAP, @@ -46,6 +46,7 @@ FILE_DATE_FMT, HUMAN_SIZES, NO, + NO_STR, OK_ARGS, OK_TMPL, SECHO_ARGS, @@ -57,7 +58,6 @@ WARN_TMPL, YES, YES_STR, - NO_STR, ) from .constants.logs import MAX_BODY_LEN from .exceptions import FormatError, ToolsError @@ -1546,18 +1546,19 @@ def bom_strip(content: t.Union[str, bytes], strip=True, bom: bytes = codecs.BOM_ content: string to remove BOM marker from if found strip: remove whitespace before & after removing BOM marker """ - content = content.strip() if strip else content + if isinstance(content, (str, bytes)): + content = content.strip() if strip else content - if isinstance(bom, bytes) and isinstance(content, str): - bom = bom.decode() - elif isinstance(bom, str) and isinstance(content, bytes): - bom = bom.encode() + if isinstance(bom, bytes) and isinstance(content, str): + bom = bom.decode() + elif isinstance(bom, str) and isinstance(content, bytes): + bom = bom.encode() - bom_len = len(bom) - if content.startswith(bom): - content = content[bom_len:] + bom_len = len(bom) + if content.startswith(bom): + content = content[bom_len:] - content = content.strip() if strip else content + content = content.strip() if strip else content return content From 470054289b46a1628670b60d61106925ec0f7bbd Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Tue, 2 May 2023 20:30:28 -0400 Subject: [PATCH 04/17] meta was incorrect previously --- axonius_api_client/api/assets/saved_query.py | 24 ++++++++++++------- .../tests_assets/test_saved_query.py | 11 +++++++++ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/axonius_api_client/api/assets/saved_query.py b/axonius_api_client/api/assets/saved_query.py index d84e88bc..449ba288 100644 --- a/axonius_api_client/api/assets/saved_query.py +++ b/axonius_api_client/api/assets/saved_query.py @@ -992,6 +992,8 @@ def build_add_model( folder: t.Optional[t.Union[str, FolderModel]] = None, create: bool = FolderDefaults.create_action, echo: bool = FolderDefaults.echo_action, + enforcement_filter: t.Optional[str] = None, + unique_adapters: bool = False, **kwargs, ) -> models.SavedQueryCreate: """Create a saved query. @@ -1057,11 +1059,11 @@ def build_add_model( create: create folder if it does not exist echo: echo folder actions to stdout/stderr sort_field_parsed: previously parsed sort field - + enforcement_filter: unknown + unique_adapters: unknown Returns: models.SavedQueryCreate: saved query dataclass to create - """ asset_scope = coerce_bool(asset_scope) private = coerce_bool(private) @@ -1106,21 +1108,25 @@ def build_add_model( fields_error=True, ) if not isinstance(sort_field_parsed, str): - sort_field_parsed: str = self.parent.fields.get_field_name(value=sort_field) or "" + sort_field_parsed: str = ( + self.parent.fields.get_field_name(value=sort_field) if sort_field else "" + ) - view_sort: dict = { - "desc": sort_descending, - "field": sort_field_parsed, + view_query_meta: dict = { + "enforcementFilter": enforcement_filter or "", + "uniqueAdapters": unique_adapters, } view_query: dict = { "filter": query or "", "expressions": expressions or [], "search": None, - "meta": {}, - "enforcementFilter": None, - "uniqueAdapters": False, + "meta": view_query_meta, "onlyExpressionsFilter": query_expr or "", } + view_sort: dict = { + "desc": sort_descending, + "field": sort_field_parsed or "", + } view: dict = { "fields": fields_parsed, "pageSize": gui_page_size, diff --git a/axonius_api_client/tests/tests_api/tests_assets/test_saved_query.py b/axonius_api_client/tests/tests_api/tests_assets/test_saved_query.py index d6c4152c..8ced0144 100644 --- a/axonius_api_client/tests/tests_api/tests_assets/test_saved_query.py +++ b/axonius_api_client/tests/tests_api/tests_assets/test_saved_query.py @@ -1095,7 +1095,18 @@ def validate_sq(asset): folder_id = query.pop("folder_id", None) assert folder_id is None or isinstance(folder_id, str) + # 2023/04/02: {'enforcementFilter': None, 'uniqueAdapters': False} + enforcement_filter = query.pop("enforcementFilter", None) + assert isinstance(enforcement_filter, str) or enforcement_filter is None + + unique_adapters = query.pop("uniqueAdapters", None) + assert isinstance(unique_adapters, bool) or unique_adapters is None + assert not query, list(query) + + # 2023/04/02: {'colExcludeAdapters': []} + col_exclude_adapters = view.pop("colExcludeAdapters", []) + assert isinstance(col_exclude_adapters, list) or col_exclude_adapters is None assert not view, list(view) document_meta = asset.pop("document_meta", {}) From 1a886c91eb47cb28883a6ba21c6fd67427778d5c Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Tue, 2 May 2023 21:59:22 -0400 Subject: [PATCH 05/17] Test fix ERROR axonius_api_client/tests/tests_api/tests_enforcements/test_tasks.py::TestTasks::test_to_dict - TypeError: unsupported operand type(s) for -: 'function' and 'datetime.datetime' --- axonius_api_client/tools.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/axonius_api_client/tools.py b/axonius_api_client/tools.py index 2a7ba878..63db28d6 100644 --- a/axonius_api_client/tools.py +++ b/axonius_api_client/tools.py @@ -2447,7 +2447,10 @@ def get_diff_seconds( seconds: t.Optional[float] = None if start is not None: start: datetime.datetime = dt_parse(obj=start) - stop: datetime.datetime = dt_now if stop is None else dt_parse(stop) + if stop is None: + stop = dt_now() + else: + stop = dt_parse(stop) delta: datetime.timedelta = stop - start seconds: float = delta.total_seconds() if isinstance(places, int): From fc9b8e1ed9a5a3759e82c4c7dda5506a9587c321 Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Wed, 3 May 2023 07:06:11 -0400 Subject: [PATCH 06/17] update history_response_schema to match REST API --- .../api/json_api/adapters/fetch_history_response.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/axonius_api_client/api/json_api/adapters/fetch_history_response.py b/axonius_api_client/api/json_api/adapters/fetch_history_response.py index d0f8aea2..413c661f 100644 --- a/axonius_api_client/api/json_api/adapters/fetch_history_response.py +++ b/axonius_api_client/api/json_api/adapters/fetch_history_response.py @@ -116,6 +116,17 @@ class AdapterFetchHistorySchema(BaseSchemaJson): load_default=None, dump_default=None, ) + has_configuration_changed = SchemaBool( + description="Shows if the configuration changed since last fetch", + load_default=False, + dump_default=False, + ) + last_fetch_time = SchemaDatetime( + description="The last fetch time", + allow_none=True, + load_default=None, + dump_default=None, + ) class Meta: """JSONAPI config.""" @@ -159,6 +170,8 @@ class AdapterFetchHistory(BaseModel): ignored_devices_count: t.Optional[int] = field_from_mm(SCHEMA, "ignored_devices_count") ignored_users_count: t.Optional[int] = field_from_mm(SCHEMA, "ignored_users_count") realtime: bool = field_from_mm(SCHEMA, "realtime") + has_configuration_changed: bool = field_from_mm(SCHEMA, "has_configuration_changed") + last_fetch_time: t.Optional[datetime.datetime] = field_from_mm(SCHEMA, "last_fetch_time") document_meta: t.Optional[dict] = dataclasses.field(default_factory=dict) From a4ee2e30d66c8c6969434b70faa0a1af059ec638 Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Wed, 3 May 2023 07:06:35 -0400 Subject: [PATCH 07/17] update adapter node connection schema to match REST API --- axonius_api_client/api/json_api/adapters/adapter_node.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/axonius_api_client/api/json_api/adapters/adapter_node.py b/axonius_api_client/api/json_api/adapters/adapter_node.py index 495e106e..f5bea596 100644 --- a/axonius_api_client/api/json_api/adapters/adapter_node.py +++ b/axonius_api_client/api/json_api/adapters/adapter_node.py @@ -257,6 +257,13 @@ class AdapterNodeCnx(BaseModel): did_notify_error: t.Optional[bool] = None note: t.Optional[t.Any] = None + # 2023/04/02 + last_successful_fetch: t.Optional[datetime.datetime] = get_field_dc_mm( + mm_field=SchemaDatetime(allow_none=True), default=None + ) + latest_configuration_change: t.Optional[datetime.datetime] = get_field_dc_mm( + mm_field=SchemaDatetime(allow_none=True), default=None + ) document_meta: t.Optional[dict] = dataclasses.field(default_factory=dict) AdapterNode: t.ClassVar[AdapterNode] = None From 05fffca559a362581a52914578cba14c46d21aed Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Wed, 3 May 2023 07:09:21 -0400 Subject: [PATCH 08/17] test fix: connection label no longer being included in connection configuration --- axonius_api_client/tests/tests_api/tests_adapters/test_cnx.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/axonius_api_client/tests/tests_api/tests_adapters/test_cnx.py b/axonius_api_client/tests/tests_api/tests_adapters/test_cnx.py index f1104ce6..180f7f29 100644 --- a/axonius_api_client/tests/tests_api/tests_adapters/test_cnx.py +++ b/axonius_api_client/tests/tests_api/tests_adapters/test_cnx.py @@ -308,8 +308,10 @@ def test_get_by_label_fail(self, apiobj): def test_get_by_label(self, apiobj): cnx = get_cnx_existing(apiobj) + # 2023/04/03 - connection_label not returned in config any longer? + label = cnx["config"].get("connection_label") or cnx.get("connection_label") or "" found = apiobj.cnx.get_by_label( - value=cnx["config"].get("connection_label") or "", + value=label, adapter_name=cnx["adapter_name"], adapter_node=cnx["node_name"], ) From 39516e4ff05d5173a18a6332bca4720ff11f934e Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Wed, 3 May 2023 07:19:21 -0400 Subject: [PATCH 09/17] lint fixes for client.{asset_type}.tags --- axonius_api_client/api/assets/labels.py | 54 +++++++++++++++++-------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/axonius_api_client/api/assets/labels.py b/axonius_api_client/api/assets/labels.py index fdbbd413..4487f6fc 100644 --- a/axonius_api_client/api/assets/labels.py +++ b/axonius_api_client/api/assets/labels.py @@ -12,11 +12,6 @@ class Labels(ChildMixins): """API for working with tags for the parent asset type. Examples: - Create a ``client`` using :obj:`axonius_api_client.connect.Connect` and assume - ``apiobj`` is either ``client.devices`` or ``client.users`` - - >>> apiobj = client.devices # or client.users - * Get all known tags: :meth:`get` * Add tags to assets: :meth:`add` * Remove tags from assets: :meth:`remove` @@ -33,6 +28,11 @@ def get(self) -> List[str]: Examples: Get all known tags for this asset type + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities >>> apiobj.labels.get() ['tag1', 'tag2'] @@ -45,6 +45,11 @@ def get_expirable_names(self) -> List[str]: Examples: Get all known expirable tags for this asset type + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities >>> apiobj.labels.get_expirable_names() ['tag1', 'tag2'] @@ -57,8 +62,13 @@ def add(self, rows: Union[List[dict], str], labels: List[str]) -> int: Examples: Get some assets to tag - >>> rows = apiobj.get(wiz_entries=[{'type': 'simple', 'value': 'name equals test'}]) - >>> len(rows) + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities + >>> data = apiobj.get(wiz_entries=[{'type': 'simple', 'value': 'name equals test'}]) + >>> len(data) 1 >>> apiobj.labels.add(rows=rows, labels=['api tag 1', 'api tag 2']) @@ -86,12 +96,15 @@ def remove(self, rows: List[dict], labels: List[str]) -> int: Examples: Get some assets to un-tag - - >>> rows = apiobj.get(wiz_entries=[{'type': 'simple', 'value': 'name equals test'}]) - >>> len(rows) + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities + >>> data = apiobj.get(wiz_entries=[{'type': 'simple', 'value': 'name equals test'}]) + >>> len(data) 1 - - >>> apiobj.labels.remove(rows=rows, labels=['api tag 1', 'api tag 2']) + >>> apiobj.labels.remove(rows=data, labels=['api tag 1', 'api tag 2']) 1 Args: @@ -101,7 +114,8 @@ def remove(self, rows: List[dict], labels: List[str]) -> int: ids = self._get_ids(rows=rows) return self._remove(labels=labels, ids=ids).value - def _get_ids(self, rows: Union[List[dict], str]) -> List[str]: + @staticmethod + def _get_ids(rows: Union[List[dict], str]) -> List[str]: """Get the internal_axon_id from a list of assets. Args: @@ -121,18 +135,24 @@ def _add(self, labels: List[str], ids: List[str]) -> json_api.generic.IntValue: entities = {"ids": listify(ids), "include": True} request_obj = api_endpoint.load_request(entities=entities, labels=listify(labels)) return api_endpoint.perform_request( - http=self.auth.http, request_obj=request_obj, asset_type=self.parent.ASSET_TYPE + http=self.auth.http, request_obj=request_obj, asset_type=self.asset_type ) + # noinspection PyUnresolvedReferences + @property + def asset_type(self) -> str: + """Get the asset type of the parent AssetMixin.""" + return self.parent.ASSET_TYPE + def _get(self) -> List[json_api.generic.StrValue]: """Direct API method to get all known labels/tags.""" api_endpoint = ApiEndpoints.assets.tags_get - return api_endpoint.perform_request(http=self.auth.http, asset_type=self.parent.ASSET_TYPE) + return api_endpoint.perform_request(http=self.auth.http, asset_type=self.asset_type) def _get_expirable_names(self) -> List[json_api.generic.StrValue]: """Direct API method to get all known expirable labels/tags.""" api_endpoint = ApiEndpoints.assets.tags_get_expirable_names - return api_endpoint.perform_request(http=self.auth.http, asset_type=self.parent.ASSET_TYPE) + return api_endpoint.perform_request(http=self.auth.http, asset_type=self.asset_type) def _remove(self, labels: List[str], ids: List[str]) -> json_api.generic.IntValue: """Direct API method to remove labels/tags from assets. @@ -146,5 +166,5 @@ def _remove(self, labels: List[str], ids: List[str]) -> json_api.generic.IntValu entities = {"ids": listify(ids), "include": True} request_obj = api_endpoint.load_request(entities=entities, labels=listify(labels)) return api_endpoint.perform_request( - http=self.auth.http, request_obj=request_obj, asset_type=self.parent.ASSET_TYPE + http=self.auth.http, request_obj=request_obj, asset_type=self.asset_type ) From 11069165262a7892dd0a243e23abb64859c075e7 Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Wed, 3 May 2023 07:36:28 -0400 Subject: [PATCH 10/17] add alias for client.{asset_type}.labels to client.{asset_type}.tags --- axonius_api_client/api/assets/asset_mixin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/axonius_api_client/api/assets/asset_mixin.py b/axonius_api_client/api/assets/asset_mixin.py index 31a86c20..4cc22a3c 100644 --- a/axonius_api_client/api/assets/asset_mixin.py +++ b/axonius_api_client/api/assets/asset_mixin.py @@ -1823,6 +1823,9 @@ def _init(self, **kwargs): self.labels: Labels = Labels(parent=self) """Work with labels (tags).""" + self.tags = self.labels + """Alias for :attr:`labels`.""" + self.saved_query: SavedQuery = SavedQuery(parent=self) """Work with saved queries.""" From 7deefbd58277ac2b9549147772d913fa927b242d Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Wed, 3 May 2023 08:20:11 -0400 Subject: [PATCH 11/17] expose "include" REST API argument Modified: - client.{asset_type}.tags.add - client.{asset_type}.tags.remove Added invert_selection: bool = False - False will add or remove tags to the supplied asset ids, True will add or remove tags to every asset id EXCEPT for the ones that are supplied. - client.{asset_type}.tags._add - client.{asset_type}.tags._remove Added include:bool = True - True will apply the changes to the supplied assets ids, False will apply the changes to every asset id except for the supplied asset ids. --- axonius_api_client/api/assets/labels.py | 86 +++++++++++++------------ 1 file changed, 46 insertions(+), 40 deletions(-) diff --git a/axonius_api_client/api/assets/labels.py b/axonius_api_client/api/assets/labels.py index 4487f6fc..197147d8 100644 --- a/axonius_api_client/api/assets/labels.py +++ b/axonius_api_client/api/assets/labels.py @@ -56,7 +56,9 @@ def get_expirable_names(self) -> List[str]: """ return [x.value for x in self._get_expirable_names()] - def add(self, rows: Union[List[dict], str], labels: List[str]) -> int: + def add( + self, rows: Union[List[dict], str], labels: List[str], invert_selection: bool = False + ) -> int: """Add tags to assets. Examples: @@ -77,21 +79,33 @@ def add(self, rows: Union[List[dict], str], labels: List[str]) -> int: Args: rows: list of internal_axon_id strs or list of assets returned from a get method labels: tags to add + invert_selection: True=add tags to assets that ARE NOT supplied in rows; + False=add tags to assets that ARE supplied in rows + """ ids = self._get_ids(rows=rows) - return self._add(labels=labels, ids=ids).value + return self._add(labels=labels, ids=ids, include=not invert_selection).value - # processed = 0 + def _add( + self, labels: List[str], ids: List[str], include: bool = True + ) -> json_api.generic.IntValue: + """Direct API method to add labels/tags to assets. - # # only do 100 labels at a time, more seems to break API - # for group in grouper(ids, 100): - # group = [x for x in group if x is not None] - # response = self._add(labels=labels, ids=group) - # processed += response + Args: + labels: tags to process + ids: internal_axon_id of assets to add tags to + include: True=add tags to assets that ARE supplied in rows; + False=add tags to assets that ARE NOT supplied in rows + """ + api_endpoint = ApiEndpoints.assets.tags_add - # return processed + entities = {"ids": listify(ids), "include": include} + request_obj = api_endpoint.load_request(entities=entities, labels=listify(labels)) + return api_endpoint.perform_request( + http=self.auth.http, request_obj=request_obj, asset_type=self.asset_type + ) - def remove(self, rows: List[dict], labels: List[str]) -> int: + def remove(self, rows: List[dict], labels: List[str], invert_selection: bool = False) -> int: """Remove tags from assets. Examples: @@ -110,34 +124,41 @@ def remove(self, rows: List[dict], labels: List[str]) -> int: Args: rows: list of internal_axon_id strs or list of assets returned from a get method labels: tags to remove - """ - ids = self._get_ids(rows=rows) - return self._remove(labels=labels, ids=ids).value - - @staticmethod - def _get_ids(rows: Union[List[dict], str]) -> List[str]: - """Get the internal_axon_id from a list of assets. + invert_selection: True=remove tags from assets that ARE NOT supplied in rows; + False=remove tags from assets that ARE supplied in rows - Args: - rows: list of internal_axon_id strs or list of assets returned from a get method """ - return [x["internal_axon_id"] if isinstance(x, dict) else x for x in listify(rows)] + ids: List[str] = self._get_ids(rows=rows) + return self._remove(labels=labels, ids=ids, include=not invert_selection).value - def _add(self, labels: List[str], ids: List[str]) -> json_api.generic.IntValue: - """Direct API method to add labels/tags to assets. + def _remove( + self, labels: List[str], ids: List[str], include: bool = True + ) -> json_api.generic.IntValue: + """Direct API method to remove labels/tags from assets. Args: labels: tags to process - ids: internal_axon_id of assets to add tags to + ids: internal_axon_id of assets to remove tags from + include: True=remove tags from assets that ARE supplied in rows; + False=remove tags from assets that ARE NOT supplied in rows """ - api_endpoint = ApiEndpoints.assets.tags_add + api_endpoint = ApiEndpoints.assets.tags_remove - entities = {"ids": listify(ids), "include": True} + entities = {"ids": listify(ids), "include": include} request_obj = api_endpoint.load_request(entities=entities, labels=listify(labels)) return api_endpoint.perform_request( http=self.auth.http, request_obj=request_obj, asset_type=self.asset_type ) + @staticmethod + def _get_ids(rows: Union[List[dict], str]) -> List[str]: + """Get the internal_axon_id from a list of assets. + + Args: + rows: list of internal_axon_id strs or list of assets returned from a get method + """ + return [x["internal_axon_id"] if isinstance(x, dict) else x for x in listify(rows)] + # noinspection PyUnresolvedReferences @property def asset_type(self) -> str: @@ -153,18 +174,3 @@ def _get_expirable_names(self) -> List[json_api.generic.StrValue]: """Direct API method to get all known expirable labels/tags.""" api_endpoint = ApiEndpoints.assets.tags_get_expirable_names return api_endpoint.perform_request(http=self.auth.http, asset_type=self.asset_type) - - def _remove(self, labels: List[str], ids: List[str]) -> json_api.generic.IntValue: - """Direct API method to remove labels/tags from assets. - - Args: - labels: tags to process - ids: internal_axon_id of assets to remove tags from - """ - api_endpoint = ApiEndpoints.assets.tags_remove - - entities = {"ids": listify(ids), "include": True} - request_obj = api_endpoint.load_request(entities=entities, labels=listify(labels)) - return api_endpoint.perform_request( - http=self.auth.http, request_obj=request_obj, asset_type=self.asset_type - ) From 8c195c9e23f43dd43c9137615e139a34e9e9d5ea Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Wed, 3 May 2023 08:32:35 -0400 Subject: [PATCH 12/17] lint fixes for asset_callbacks.Base --- .../api/asset_callbacks/base.py | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/axonius_api_client/api/asset_callbacks/base.py b/axonius_api_client/api/asset_callbacks/base.py index df52e375..cd347393 100644 --- a/axonius_api_client/api/asset_callbacks/base.py +++ b/axonius_api_client/api/asset_callbacks/base.py @@ -38,21 +38,18 @@ ) +# noinspection SpellCheckingInspection def crjoin(value): """Pass.""" joiner = "\n - " return joiner + joiner.join(value) +# noinspection PyProtectedMember,PyUnresolvedReferences,PyAttributeOutsideInit class Base: """Callbacks for formatting asset data. Examples: - Create a ``client`` using :obj:`axonius_api_client.connect.Connect` and assume - ``apiobj`` is either ``client.devices`` or ``client.users`` - - >>> apiobj = client.devices # or client.users - * :meth:`args_map` for callback generic arguments to format assets. * :meth:`args_map_custom` for callback specific arguments to format and export data. @@ -66,7 +63,11 @@ def args_map(cls) -> dict: Create a ``client`` using :obj:`axonius_api_client.connect.Connect` and assume ``apiobj`` is either ``client.devices`` or ``client.users`` - >>> apiobj = client.devices # or client.users + >>> import axonius_api_client as axonapi + >>> connect_args: dict = axonapi.get_env_connect() + >>> client: axonapi.Connect = axonapi.Connect(**connect_args) + >>> apiobj: axonapi.api.assets.AssetMixin = client.devices + >>> # or client.users or client.vulnerabilities Flatten complex fields - Will take all sub-fields of complex fields and put them on the root level with their values index correlated to each other. @@ -196,7 +197,7 @@ def args_map_base(cls) -> dict: "csv_field_null": True, } - def get_arg_value(self, arg: str) -> Union[str, list, bool, int]: + def get_arg_value(self, arg: str) -> t.Any: """Get an argument value. Args: @@ -204,7 +205,7 @@ def get_arg_value(self, arg: str) -> Union[str, list, bool, int]: """ return self.GETARGS.get(arg, self.args_map()[arg]) - def set_arg_value(self, arg: str, value: Union[str, list, bool, int]): + def set_arg_value(self, arg: str, value: t.Any): """Set an argument value. Args: @@ -267,8 +268,8 @@ def start(self, **kwargs): self.echo(msg=f"Adding fields {missing} to field_excludes: {excludes}", debug=True) self.set_arg_value("field_excludes", value=excludes + missing) - cbargs = crjoin(join_kv(obj=self.GETARGS)) - self.LOG.debug(f"Get Extra Arguments: {cbargs}") + cb_args = crjoin(join_kv(obj=self.GETARGS)) + self.LOG.debug(f"Get Extra Arguments: {cb_args}") config = crjoin(self.args_strs) self.echo(msg=f"Configuration: {config}") @@ -276,6 +277,7 @@ def start(self, **kwargs): store = crjoin(join_kv(obj=self.STORE)) self.echo(msg=f"Get Arguments: {store}") + # noinspection PyUnusedLocal def echo_columns(self, **kwargs): """Echo the columns of the fields selected.""" if getattr(self, "ECHO_DONE", False): @@ -382,6 +384,8 @@ def do_row(self, rows: Union[List[dict], dict]) -> List[dict]: """ debug_timing = self.get_arg_value("debug_timing") + p_start = None + cb_start = None if debug_timing: # pragma: no cover p_start = dt_now() @@ -391,11 +395,11 @@ def do_row(self, rows: Union[List[dict], dict]) -> List[dict]: rows = cb(rows=rows) # print(f"{cb} {json_dump(rows)}") - if debug_timing: # pragma: no cover + if debug_timing and cb_start: # pragma: no cover cb_delta = dt_now() - cb_start self.LOG.debug(f"CALLBACK {cb} took {cb_delta} for {len(rows)} rows") - if debug_timing: # pragma: no cover + if debug_timing and p_start: # pragma: no cover p_delta = dt_now() - p_start self.LOG.debug(f"CALLBACKS TOOK {p_delta} for {len(rows)} rows") @@ -531,7 +535,8 @@ def _do_join_values(self, row: dict): if trim_len and isinstance(value, str) and len(value) >= trim_len: field_len = len(value) msg = trim_str.format(field_len=field_len, trim_len=trim_len) - row[field] = value = joiner.join([value[:trim_len], msg]) + value = [value[:trim_len], msg] + row[field] = joiner.join(value) def do_change_field_replace(self, rows: Union[List[dict], dict]) -> List[dict]: """Asset callback to replace characters. @@ -558,6 +563,7 @@ def field_replacements(self) -> List[Tuple[str, str]]: """Parse the supplied list of field name replacements.""" def parse_replace(replace): + """Parse the supplied list of field name replacements.""" if isinstance(replace, str): replace = replace.split("=", maxsplit=1) @@ -593,7 +599,6 @@ def _field_compress(self, key: str) -> str: return key splits = key.split(".") - prefix = "" if splits[0] == "specific_data": prefix = AGG_ADAPTER_NAME @@ -715,6 +720,7 @@ def _do_explode_entities(self, row: dict) -> List[dict]: """ def explode(idx: int, adapter: str) -> dict: + """Explode a row into a row for each asset entity.""" new_row = {"adapters": adapter} for k, v in row.items(): @@ -985,7 +991,7 @@ def echo( self, msg: str, debug: bool = False, - error: Union[bool, t.Type[Exception]] = False, + error: Union[bool, str, t.Type[Exception]] = False, warning: bool = False, level: str = "info", level_debug: str = "debug", @@ -1000,9 +1006,11 @@ def echo( error: message is an error warning: message is a warning level: logging level for non error/non warning messages + level_debug: logging level for debug messages level_error: logging level for error messages level_warning: logging level for warning messages abort: sys.exit(1) if error is true + debug: message is a debug message """ do_echo = self.get_arg_value("do_echo") @@ -1086,6 +1094,7 @@ def final_columns(self) -> List[str]: """Get the columns that will be returned.""" def get_key(s): + """Get the key for a schema.""" return self._field_replace(self._field_compress(s[key])) if hasattr(self, "_final_columns"): @@ -1232,7 +1241,7 @@ def __repr__(self) -> str: STORE: dict = None """store dict used by get assets method to track arguments.""" - CURRENT_ROWS: None + CURRENT_ROWS: t.Optional[t.List[dict]] = None """current rows being processed""" GETARGS: dict = None @@ -1248,6 +1257,7 @@ def __repr__(self) -> str: """tracker of custom callbacks that have been executed by :meth:`do_custom_cbs`""" +# noinspection PyAttributeOutsideInit class ExportMixins(Base): """Export mixins for callbacks.""" @@ -1316,7 +1326,7 @@ def open_fd_path(self) -> IO: debug=True, ) elif not export_overwrite: - msg = f"Export file {str(self._file_path)!r} already exists and overwite is False!" + msg = f"Export file {str(self._file_path)!r} already exists and overwrite is False!" self.echo(msg=msg, error=ApiError, level="error") else: self._file_mode: str = "Overwrote existing file" From a2b8c9757b25b998e1bac01ab6825c541680a2a7 Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Wed, 3 May 2023 08:36:22 -0400 Subject: [PATCH 13/17] Add tags_add_invert_selection and tags_remove_invert_selection to asset_callbacks.Base --- axonius_api_client/api/asset_callbacks/base.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/axonius_api_client/api/asset_callbacks/base.py b/axonius_api_client/api/asset_callbacks/base.py index cd347393..7109f7ce 100644 --- a/axonius_api_client/api/asset_callbacks/base.py +++ b/axonius_api_client/api/asset_callbacks/base.py @@ -183,7 +183,9 @@ def args_map_base(cls) -> dict: "field_null_value": None, "field_null_value_complex": [], "tags_add": [], + "tags_add_invert_selection": False, "tags_remove": [], + "tags_remove_invert_selection": False, "report_adapters_missing": False, "report_software_whitelist": [], "page_progress": 10000, @@ -804,17 +806,23 @@ def do_tag_add(self): """Add tags to assets.""" tags_add = listify(self.get_arg_value("tags_add")) rows_add = self.TAG_ROWS_ADD + invert_selection = self.get_arg_value("tags_add_invert_selection") if tags_add and rows_add: self.echo(msg=f"Adding tags {tags_add} to {len(rows_add)} assets") - self.APIOBJ.labels.add(rows=rows_add, labels=tags_add) + self.APIOBJ.labels.add( + rows=rows_add, labels=tags_add, invert_selection=invert_selection + ) def do_tag_remove(self): """Remove tags from assets.""" tags_remove = listify(self.get_arg_value("tags_remove")) rows_remove = self.TAG_ROWS_REMOVE + invert_selection = self.get_arg_value("tags_remove_invert_selection") if tags_remove and rows_remove: self.echo(msg=f"Removing tags {tags_remove} from {len(rows_remove)} assets") - self.APIOBJ.labels.remove(rows=rows_remove, labels=tags_remove) + self.APIOBJ.labels.remove( + rows=rows_remove, labels=tags_remove, invert_selection=invert_selection + ) def process_tags_to_add(self, rows: Union[List[dict], dict]) -> List[dict]: """Add assets to tracker for adding tags. @@ -1420,7 +1428,9 @@ def arg_export_fd_close(self) -> bool: "field_null_value": "Null value to use for missing simple fields", "field_null_value_complex": "Null value to use for missing complex fields", "tags_add": "Tags to add to assets", + "tags_add_invert_selection": "Invert selection for tags to add", "tags_remove": "Tags to remove from assets", + "tags_remove_invert_selection": "Invert selection for tags to remove", "report_adapters_missing": "Add Missing Adapters calculation", "report_software_whitelist": "Missing Software to calculate", "page_progress": "Echo page progress every N assets", From ef753e62e4c8516ccf55e3185e9450c9d7ecf49f Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Wed, 3 May 2023 08:58:55 -0400 Subject: [PATCH 14/17] Add axonshell options to axonshell {asset_type} get* These map to the new asset_callback arguments tags_add_invert_selection and tags_remove_invert_selection: ``` --untag-invert / --no-untag-invert Only remove tags from assets that do NOT match the query provided [env var: AX_TAGS_REMOVE_INVERT_SELECTION; default: no-untag-invert] ``` --- .../cli/grp_assets/grp_common.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/axonius_api_client/cli/grp_assets/grp_common.py b/axonius_api_client/cli/grp_assets/grp_common.py index 8ee0a55e..77fad184 100644 --- a/axonius_api_client/cli/grp_assets/grp_common.py +++ b/axonius_api_client/cli/grp_assets/grp_common.py @@ -491,6 +491,16 @@ def wiz_callback(ctx, param, value): hidden=False, metavar="TAG", ), + click.option( + "--tag-invert/--no-tag-invert", + "tags_add_invert_selection", + default=asset_callbacks.Base.args_map()["tags_add_invert_selection"], + help="Only add tags to assets that do NOT match the query provided", + show_envvar=True, + show_default=True, + is_flag=True, + hidden=False, + ), click.option( "--untag", "tags_remove", @@ -502,6 +512,16 @@ def wiz_callback(ctx, param, value): hidden=False, metavar="TAG", ), + click.option( + "--untag-invert/--no-untag-invert", + "tags_remove_invert_selection", + default=asset_callbacks.Base.args_map()["tags_remove_invert_selection"], + help="Only remove tags from assets that do NOT match the query provided", + show_envvar=True, + show_default=True, + is_flag=True, + hidden=False, + ), click.option( "--include-details/--no-include-details", "-id/-nid", From 2c5af6f43b16452c283cc5cb86f295fbeeab03e4 Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Wed, 3 May 2023 08:59:47 -0400 Subject: [PATCH 15/17] Add more info to echo Show the number of assets supplied, number of tags supplied (and the tags), invert_selection, and finally, the number of assets that were modified --- .../api/asset_callbacks/base.py | 53 +++++++++++++------ 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/axonius_api_client/api/asset_callbacks/base.py b/axonius_api_client/api/asset_callbacks/base.py index 7109f7ce..450fd216 100644 --- a/axonius_api_client/api/asset_callbacks/base.py +++ b/axonius_api_client/api/asset_callbacks/base.py @@ -45,7 +45,7 @@ def crjoin(value): return joiner + joiner.join(value) -# noinspection PyProtectedMember,PyUnresolvedReferences,PyAttributeOutsideInit +# noinspection PyProtectedMember,PyAttributeOutsideInit class Base: """Callbacks for formatting asset data. @@ -804,25 +804,46 @@ def do_tagging(self): def do_tag_add(self): """Add tags to assets.""" - tags_add = listify(self.get_arg_value("tags_add")) - rows_add = self.TAG_ROWS_ADD + tags = listify(self.get_arg_value("tags_add")) + rows = self.TAG_ROWS_ADD invert_selection = self.get_arg_value("tags_add_invert_selection") - if tags_add and rows_add: - self.echo(msg=f"Adding tags {tags_add} to {len(rows_add)} assets") - self.APIOBJ.labels.add( - rows=rows_add, labels=tags_add, invert_selection=invert_selection + count_tags = len(tags) + count_supplied = len(rows) + msgs = [ + f" Tags supplied ({count_tags}): {tags}", + f" Asset IDs supplied ({count_supplied})", + f" Invert selection: {invert_selection}", + ] + if tags: + self.echo(["Performing API call to add tags to assets", *msgs]) + count_modified = self.APIOBJ.labels.add( + rows=rows, labels=tags, invert_selection=invert_selection ) + self.echo(msg=[f"API added tags to {count_modified} assets", *msgs]) def do_tag_remove(self): """Remove tags from assets.""" - tags_remove = listify(self.get_arg_value("tags_remove")) - rows_remove = self.TAG_ROWS_REMOVE + tags = listify(self.get_arg_value("tags_remove")) + rows = self.TAG_ROWS_REMOVE invert_selection = self.get_arg_value("tags_remove_invert_selection") - if tags_remove and rows_remove: - self.echo(msg=f"Removing tags {tags_remove} from {len(rows_remove)} assets") - self.APIOBJ.labels.remove( - rows=rows_remove, labels=tags_remove, invert_selection=invert_selection + count_tags = len(tags) + count_supplied = len(rows) + msgs = [ + f" Asset IDs supplied ({count_supplied})", + f" Tags supplied ({count_tags}): {tags}", + f" Invert selection: {invert_selection}", + ] + if tags: + self.echo(["Performing API call to remove tags from assets", *msgs]) + count_modified = self.APIOBJ.labels.remove( + rows=rows, labels=tags, invert_selection=invert_selection ) + msgs = [ + f"API finished removing tags from assets", + f" Asset IDs modified: " f"{count_modified}", + *msgs, + ] + self.echo(msg=msgs) def process_tags_to_add(self, rows: Union[List[dict], dict]) -> List[dict]: """Add assets to tracker for adding tags. @@ -997,7 +1018,7 @@ def excluded_schemas(self) -> List[dict]: def echo( self, - msg: str, + msg: t.Union[str, t.List[str]], debug: bool = False, error: Union[bool, str, t.Type[Exception]] = False, warning: bool = False, @@ -1021,7 +1042,7 @@ def echo( debug: message is a debug message """ do_echo = self.get_arg_value("do_echo") - + msg = "\n".join(listify(msg)) if do_echo: if error: echo_error(msg=msg, abort=abort) @@ -1184,7 +1205,7 @@ def schema_to_explode(self) -> dict: def adapter_map(self) -> dict: """Build a map of adapters that have connections.""" if getattr(self, "_adapter_map", None): - return self._adapter_map + return getattr(self, "_adapter_map", None) self._adapters_meta = getattr( self, "_adapters_meta", self.APIOBJ.adapters.get(get_clients=False) From 0640f59eb4fa09f11b9edc229fe492c49eb96163 Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Wed, 3 May 2023 09:11:57 -0400 Subject: [PATCH 16/17] bugfix: wrong type defined in AssetGetByIdSchema i blame the rest api, i copied the definition from there. --- axonius_api_client/api/json_api/assets/asset_id_response.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/axonius_api_client/api/json_api/assets/asset_id_response.py b/axonius_api_client/api/json_api/assets/asset_id_response.py index d1948877..8335bf4d 100644 --- a/axonius_api_client/api/json_api/assets/asset_id_response.py +++ b/axonius_api_client/api/json_api/assets/asset_id_response.py @@ -52,7 +52,7 @@ class AssetByIdSchema(BaseSchemaJson): description="Data for this asset", ) labels = mm_fields.List( - mm_fields.Dict(), + mm_fields.Str(), load_default=list, dump_default=list, allow_none=True, @@ -110,7 +110,7 @@ class AssetById(BaseModel): basic: dict = field_from_mm(SCHEMA, "basic") aggregated_specific_data: dict = field_from_mm(SCHEMA, "aggregated_specific_data") data: t.List[dict] = field_from_mm(SCHEMA, "data") - labels: t.List[dict] = field_from_mm(SCHEMA, "labels") + labels: t.List[str] = field_from_mm(SCHEMA, "labels") expirable_tags: t.List[dict] = field_from_mm(SCHEMA, "expirable_tags") labels_metadata: t.List[dict] = field_from_mm(SCHEMA, "labels_metadata") compliance_meta: dict = field_from_mm(SCHEMA, "compliance_meta") From 1325ff07d9cce54f9e93a560e21264f18ff640db Mon Sep 17 00:00:00 2001 From: Jim Olsen Date: Wed, 3 May 2023 10:04:26 -0400 Subject: [PATCH 17/17] test fix for when data scopes are not enabled --- .../tests/tests_api/tests_system/test_data_scopes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/axonius_api_client/tests/tests_api/tests_system/test_data_scopes.py b/axonius_api_client/tests/tests_api/tests_system/test_data_scopes.py index 5269e684..f4ebd995 100644 --- a/axonius_api_client/tests/tests_api/tests_system/test_data_scopes.py +++ b/axonius_api_client/tests/tests_api/tests_system/test_data_scopes.py @@ -69,6 +69,8 @@ def create_data_scope(self, apiobj, name, device_scopes, user_scopes): self.cleanup_data_scopes(apiobj=apiobj, value=row.uuid) def create_asset_scope(self, apiasset, name): + if not apiasset.data_scopes.is_feature_enabled: + pytest.skip("Data Scopes Feature Flag not enabled") self.cleanup_asset_scopes(apiasset=apiasset, value=name) row = apiasset.saved_query.add(