Skip to content
Permalink
Browse files

implemented index alias to make import atomic

  • Loading branch information
garbas committed May 22, 2020
1 parent b264d7a commit e95270633a63b93e00931c26b729da1b89cfd5ac
Showing with 79 additions and 71 deletions.
  1. +77 −69 scripts/import-channel
  2. +1 −1 src/Page/Options.elm
  3. +1 −1 src/Page/Packages.elm
@@ -29,8 +29,7 @@ click_log.basic_config(logger)


CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))


INDEX_SCHEMA_VERSION=1
ANALYSIS = {
"analyzer": {
"nixAttrName": {
@@ -70,6 +69,47 @@ ANALYSIS = {
},
},
}
PACKAGES_MAPPING = dict(
properties=dict(
attr_name=dict(
type="text",
analyzer="nixAttrName",
fields={"raw": {"type": "keyword"}},
),
attr_set=dict(type="keyword"),
pname=dict(type="keyword"),
pversion=dict(type="keyword"),
description=dict(type="text"),
longDescription=dict(type="text"),
license=dict(
type="nested",
properties=dict(
fullName=dict(type="text"), url=dict(type="text"),
),
),
maintainers=dict(
type="nested",
properties=dict(
name=dict(type="text"),
email=dict(type="text"),
github=dict(type="text"),
),
),
platforms=dict(type="keyword"),
position=dict(type="text"),
homepage=dict(type="keyword"),
),
)
OPTIONS_MAPPING = dict(
properties=dict(
option_name=dict(type="keyword"),
description=dict(type="text"),
type=dict(type="keyword"),
default=dict(type="text"),
example=dict(type="text"),
source=dict(type="keyword"),
),
)


def get_last_evaluation(channel):
@@ -235,75 +275,34 @@ def get_options(evaluation):
return len(options), gen


def recreate_index(es, channel):
packages_index = f"{channel}-packages"
if es.indices.exists(packages_index):
es.indices.delete(index=packages_index)
logger.debug(
f"recreate_index: index '{packages_index}' already exists and was deleted"
)
def create_index(es, index, mapping):
if es.indices.exists(index):
logger.debug(f"create_index: index '{index}' already exists")
return
es.indices.create(
index=packages_index,
body=dict(
settings=dict(number_of_shards=1, analysis=ANALYSIS),
mappings=dict(
properties=dict(
attr_name=dict(
type="text",
analyzer="nixAttrName",
fields={"raw": {"type": "keyword"}},
),
attr_set=dict(type="keyword"),
pname=dict(type="keyword"),
pversion=dict(type="keyword"),
description=dict(type="text"),
longDescription=dict(type="text"),
license=dict(
type="nested",
properties=dict(
fullName=dict(type="text"), url=dict(type="text"),
),
),
maintainers=dict(
type="nested",
properties=dict(
name=dict(type="text"),
email=dict(type="text"),
github=dict(type="text"),
),
),
platforms=dict(type="keyword"),
position=dict(type="text"),
homepage=dict(type="keyword"),
),
),
),
index=index,
body={
"settings": {
"number_of_shards": 1,
"analysis": ANALYSIS,
},
"mappings": mapping,
},
)
logger.debug(f"recreate_index: index '{packages_index}' was created")

options_index = f"{channel}-options"
if es.indices.exists(options_index):
es.indices.delete(index=options_index)
logger.debug(
f"recreate_index: index '{options_index}' already exists and was deleted"
)
es.indices.create(
index=options_index,
body=dict(
settings=dict(number_of_shards=1, analysis=ANALYSIS),
mappings=dict(
properties=dict(
option_name=dict(type="keyword"),
description=dict(type="text"),
type=dict(type="keyword"),
default=dict(type="text"),
example=dict(type="text"),
source=dict(type="keyword"),
),
),
),
logger.debug(f"create_index: index '{index}' was created")


def create_index_name(type_, channel, evaluation):
return (
f"latest-{channel}-{type_}",
f"evaluation-{INDEX_SCHEMA_VERSION}-{channel}-{evaluation['revisions_since_start']}-{evaluation['git_revision']}-{type_}",
)
logger.debug(f"recreate_index: index '{options_index}' was created")


def update_alias(es, name, index):
tmp = es.indices.put_alias(index=index, name=name)
debugger
logger.debug(f"'{name}' alias now points to '{index}' index")


@click.command()
@@ -324,7 +323,12 @@ def main(es_url, channel, verbose):

evaluation = get_last_evaluation(channel)
es = elasticsearch.Elasticsearch([es_url])
recreate_index(es, channel)

# ensure indexes exist
packages_alias, packages_index = create_index_name("packages", channel, evaluation)
options_alias, options_index = create_index_name("options", channel, evaluation)
create_index(es, packages_index, PACKAGES_MAPPING)
create_index(es, options_index, OPTIONS_MAPPING)

# write packages
number_of_packages, gen_packages = get_packages(evaluation)
@@ -352,6 +356,10 @@ def main(es_url, channel, verbose):
successes += ok
print("Indexed %d/%d options" % (successes, number_of_options))

# update alias
update_alias(es, packages_alias, packages_index)
update_alias(es, options_alias, options_index)


if __name__ == "__main__":
main()
@@ -219,7 +219,7 @@ makeRequest :
makeRequest options channel query from size =
ElasticSearch.makeRequest
"option_name"
("nixos-" ++ channel ++ "-options")
("latest-nixos-" ++ channel ++ "-options")
decodeResultItemSource
options
query
@@ -287,7 +287,7 @@ makeRequest :
makeRequest options channel query from size =
ElasticSearch.makeRequest
"attr_name"
("nixos-" ++ channel ++ "-packages")
("latest-nixos-" ++ channel ++ "-packages")
decodeResultItemSource
options
query

0 comments on commit e952706

Please sign in to comment.
You can’t perform that action at this time.