diff --git a/CHANGELOG.md b/CHANGELOG.md index 796d4f1..6804b4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +Version 2.1.0 + + Added a system that gathers all non-Keyfactor friendly characters and allows the user to configure an alternative. + Added pagination based batch processing, memory consumption has been drastically reduced. + Version 2.0.3 Added a setting to enable or disable syncing deactivated custom fields from DigiCert. diff --git a/README.md b/README.md index 3ccc863..72f6b2a 100644 --- a/README.md +++ b/README.md @@ -50,11 +50,17 @@ This should include the Keyfactor API endpoint, of the format https://domain.com This should include the common prefix all DigiCert certs have in your Keyfactor instance. For example, "DigiCert" - ImportAllCustomDigicertFields This setting enables the tool to import all of the custom metadata fields included in DigiCert and sync all of their data. -- ReplaceDigicertWhiteSpaceCharacterInName -In case the ImportAllCustomDigicertFields setting is used, this is necessary to for metadata field label conversion. DigiCert supports spaces in labels and Keyfactor does not, so this replaces the spaces in the name with your character sequence of choice. + +During the first run, the tool will scan the custom fields it will be importing for characters that are not supported in Keyfactor Metadata field names. +Each unsupported character will be shown in a file named "replacechar.json" and its replacement can be selected. If the values in the file are not populated, the tool will not run a second time. - ImportDataForDeactivatedDigiCertFields If this is enabled, custom metadata fields that were deactivated in DigiCert will also be synced, and the data stored in these fields in certificates will be too. +### replacechar.json settings +This file is populated during the first run of the tool if the ImportAllCustomDigicertFields setting is toggled. +The only text that needs replacing is shown as "null", and can be filled with any alphanumeric string. The "_" and "-" characters are also supported. + + ### manualfields.json settings This file is used to specify which metadata fields should be synced up. diff --git a/digicert-metadata-sync/BannedCharacters.cs b/digicert-metadata-sync/BannedCharacters.cs new file mode 100644 index 0000000..74e33fe --- /dev/null +++ b/digicert-metadata-sync/BannedCharacters.cs @@ -0,0 +1,68 @@ +// Copyright 2021 Keyfactor +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text.RegularExpressions; +using Newtonsoft.Json.Linq; + +namespace DigicertMetadataSync; + +internal partial class DigicertSync +{ + public static List BannedCharacterParse(string input) + { + string pattern = "[a-zA-Z0-9-_]"; + + List bannedChars = new List(); + + foreach (char c in input) + { + if (!Regex.IsMatch(c.ToString(), pattern)) + { + CharDBItem localitem = new CharDBItem(); + localitem.character = c.ToString(); + localitem.replacementcharacter = "null"; + bannedChars.Add(localitem); + } + } + + if (bannedChars.Count > 0) + { + Console.WriteLine("The field name " + input + " contains the following invalid characters: " + + string.Join("", bannedChars.Select(item => item.character))); + } + else + { + Console.WriteLine("The field name " + input + " is valid."); + } + + return bannedChars; + } + + public static void CheckForChars(List input, List allBannedChars, bool restartandconfigrequired) + { + foreach (var dgfield in input) + { + List newChars = BannedCharacterParse(dgfield.DigicertFieldName); + foreach (var newchar in newChars) + { + bool exists = allBannedChars.Any(allcharchar => allcharchar.character == newchar.character); + if (!exists) + { + allBannedChars.Add(newchar); + restartandconfigrequired = true; + } + } + } + } +} diff --git a/digicert-metadata-sync/Helpers.cs b/digicert-metadata-sync/Helpers.cs index 6dd9d45..0246749 100644 --- a/digicert-metadata-sync/Helpers.cs +++ b/digicert-metadata-sync/Helpers.cs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. + +using System.Collections.Generic; + using System.Text.RegularExpressions; using Newtonsoft.Json.Linq; @@ -54,9 +57,13 @@ public static Dictionary ClassConverter(object obj) return null; } - public static string ReplaceAllWhiteSpaces(string str, string replacement) + public static string ReplaceAllBannedCharacters(string input, ListallBannedChars) { - return Regex.Replace(str, @"\s+", "_-_"); + foreach (CharDBItem item in allBannedChars) + { + input = input.Replace(item.character, item.replacementcharacter); + } + return input; } public static bool CheckMode(string mode) @@ -65,17 +72,18 @@ public static bool CheckMode(string mode) return false; } - private static List convertlisttokf(List inputlist, - string replacementcharacter) + private static List convertlisttokf(List inputlist, List allBannedChars, bool importallcustomfields) { var formattedlist = new List(); if (inputlist.Count != 0) foreach (var input in inputlist) { var formatinstance = new KeyfactorMetadataInstanceSendoff(); - if (input.KeyfactorMetadataFieldName == null || input.KeyfactorMetadataFieldName == "") - //If name is emtpy, use autocomplete. - formatinstance.Name = ReplaceAllWhiteSpaces(input.DigicertFieldName, replacementcharacter); + + if (input.KeyfactorMetadataFieldName == null || input.KeyfactorMetadataFieldName == "" || input.FieldType == "Custom") + //If name is empty, clean up the characters. + formatinstance.Name = ReplaceAllBannedCharacters(input.DigicertFieldName, allBannedChars); + else //Use user input preferred name. formatinstance.Name = input.KeyfactorMetadataFieldName; @@ -86,6 +94,9 @@ private static List convertlisttokf(List>(rawresponse, - new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - Console.WriteLine("Got DigiCert issued certs from keyfactor"); - _logger.Debug("Got DigiCert issued certs from keyfactor"); //Getting list of custom metadata fields from Keyfactor var getmetadalistkf = keyfactorapilocation + "MetadataFields"; @@ -85,6 +78,7 @@ public static void Main(string[] args) var config = new ConfigurationBuilder().SetBasePath(AppDomain.CurrentDomain.BaseDirectory) .AddJsonFile("manualfields.json").Build(); var kfcustomfields = new List(); + if (importallcustomdigicertfields) { _logger.Debug("Loading custom fields using autofill"); @@ -105,8 +99,7 @@ WHITESPACE MUST BE REMOVED FROM THE NAME. CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. */ localkffieldinstance.DigicertFieldName = customdigicertmetadatafieldlist[i].label; - localkffieldinstance.KeyfactorMetadataFieldName = - ReplaceAllWhiteSpaces(customdigicertmetadatafieldlist[i].label, replacementcharacter); + localkffieldinstance.KeyfactorMetadataFieldName = customdigicertmetadatafieldlist[i].label; _logger.Debug("DC field name {0} becomes {1} in Keyfactor", localkffieldinstance.DigicertFieldName, localkffieldinstance.KeyfactorMetadataFieldName); } @@ -137,12 +130,21 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. if (kfcustomfields == null) kfcustomfields = new List(); _logger.Debug("Loading custom fields using json, no autofill/conversion"); } + foreach (var item in kfcustomfields) + { + item.FieldType = "Custom"; + } + //Adding metadata fields for the ID and the email of the requester from DigiCert. var kfmanualfields = new List(); var manualfieldslist = "ManualFields"; kfmanualfields = config.GetSection(manualfieldslist).Get>(); if (kfmanualfields == null) kfmanualfields = new List(); + foreach (var item in kfmanualfields) + { + item.FieldType = "Manual"; + } _logger.Debug("Performed field conversion."); //Pulling list of existing metadata fields from Keyfactor for later comparison. @@ -167,9 +169,52 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. Console.WriteLine("Pulled existing metadata fields from keyfactor."); _logger.Debug("Pulled existing metadata fields from Keyfactor."); - // Converting the read in fields into sendable lists - var convertedmanualfields = convertlisttokf(kfmanualfields, replacementcharacter); - var convertedcustomfields = convertlisttokf(kfcustomfields, replacementcharacter); + + + + // Carrying out the persistent banned character database check + + // Loading up the character database + string currentDirectory = Directory.GetCurrentDirectory(); + + string filePath = Path.Combine(currentDirectory, "replacechar.json"); + + bool restartandconfigrequired = false; + + List allBannedChars = JsonConvert.DeserializeObject>(File.ReadAllText(filePath)); + + if (importallcustomdigicertfields) + { + CheckForChars(kfmanualfields, allBannedChars, restartandconfigrequired); + CheckForChars(kfcustomfields, allBannedChars, restartandconfigrequired); + + string formattedjsonchars = JsonConvert.SerializeObject(allBannedChars, Formatting.Indented); + File.WriteAllText(filePath, formattedjsonchars); + + foreach (var badchar in allBannedChars) + { + if (badchar.replacementcharacter == "null") + { + restartandconfigrequired = true; + break; + } + } + + if (restartandconfigrequired) + { + _logger.Trace("Please replace \"null\" with your desired replacement characters in replacechar.json and re-run the tool! Only alphanumerics, \"-\" and \"_\" are allowed"); + Console.WriteLine("Please replace \"null\" with your desired replacement characters in replacechar.json and re-run the tool! Only alphanumerics, \"-\" and \"_\" are allowed"); + Environment.Exit(0); + } + + + } + + // Converting the read in fields into sendable lists + var convertedmanualfields = convertlisttokf(kfmanualfields, allBannedChars, importallcustomdigicertfields); + var convertedcustomfields = convertlisttokf(kfcustomfields, allBannedChars, importallcustomdigicertfields); + + _logger.Trace("Sending following manual fields to KF: {0}", JsonConvert.SerializeObject(convertedmanualfields)); var totalfieldsadded = 0; @@ -186,311 +231,358 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. totalfieldsadded += customresult.Item1; var allnewfields = manualresult.Item2.Concat(customresult.Item2).ToList(); + + + //Processing this batch + Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + _logger.Debug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + // Syncing Data from Keyfactor TO DigiCert // Sync from DigiCert to Keyfactor must run at least once prior to this - only runs with custom fields if (config_mode == "kftodc") { - Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - _logger.Debug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - - var fullcustomdgfieldlist = new List(); - var newcustomfieldsfordg = new List(); - // Rebuild the list of metadata field names as they are on DigiCerts side. - - // This covers all of the custom fields on Digicerts side - foreach (var dgcustomfield in customdigicertmetadatafieldlist) + // Initialize variable to keep track of items downloaded so far + int certsdownloaded = 0; + var certcounttracker = 0; + var totalcertsprocessed = 0; + var numcertsdatauploaded = 0; + for (int batchnum = 0; batchnum < numberOfBatches; batchnum++) { - var localdigicertfieldinstance = new DigicertCustomFieldInstance(); + // Check if reaching the arbitrary limit + if (certsdownloaded + batchsize > returnlimitint) + { + Console.WriteLine($"Stopped downloading at the configured limit of {returnlimitint} items."); + _logger.Debug($"Stopped downloading at the configured limit of {returnlimitint} items."); + break; + } - localdigicertfieldinstance.label = dgcustomfield.label; - localdigicertfieldinstance.is_active = dgcustomfield.is_active; - localdigicertfieldinstance.data_type = dgcustomfield.data_type; - localdigicertfieldinstance.is_required = dgcustomfield.is_required; - foreach (var kffieldeq in kfcustomfields) - if (dgcustomfield.label == kffieldeq.DigicertFieldName) - localdigicertfieldinstance.kf_field_name = kffieldeq.DigicertFieldName; + var fullcustomdgfieldlist = new List(); + var newcustomfieldsfordg = new List(); + + // Download the items in this batch + var digicertlookup = keyfactorapilocation + "Certificates?pq.queryString=IssuerDN%20-contains%20%22" + + digicertIssuerQueryterm + "%22&pq.returnLimit=" + batchsize.ToString() + + "&includeMetadata=true" + "&pq.pageReturned=" + batchnum.ToString(); + var request = new RestRequest(digicertlookup); + request.AddHeader("Accept", "application/json"); + request.AddHeader("x-keyfactor-api-version", "1"); + request.AddHeader("x-keyfactor-requested-with", "APIClient"); + var response = client.Execute(request); + var rawresponse = response.Content; + var certlist = JsonConvert.DeserializeObject>(rawresponse, + new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + Console.WriteLine("Got DigiCert issued certs from keyfactor"); + _logger.Debug("Got DigiCert issued certs from keyfactor"); - fullcustomdgfieldlist.Add(localdigicertfieldinstance); - } + // Rebuild the list of metadata field names as they are on DigiCerts side. + // This covers all of the custom fields on Digicerts side + foreach (var dgcustomfield in customdigicertmetadatafieldlist) + { + var localdigicertfieldinstance = new DigicertCustomFieldInstance(); - //This covers all of the new fields on Keyfactors side, including new ones - needs to have digicert ids for the new ones - foreach (var kfcustomfield in kfcustomfields) - { - var localdigicertfieldinstance = new DigicertCustomFieldInstance(); - localdigicertfieldinstance.label = kfcustomfield.DigicertFieldName; - localdigicertfieldinstance.is_active = true; - localdigicertfieldinstance.kf_field_name = kfcustomfield.KeyfactorMetadataFieldName; - if (kfcustomfield.KeyfactorDataType == "String") - localdigicertfieldinstance.data_type = "text"; - else if (kfcustomfield.KeyfactorDataType == "Int") - localdigicertfieldinstance.data_type = "int"; - else - localdigicertfieldinstance.data_type = "anything"; - localdigicertfieldinstance.is_required = false; + localdigicertfieldinstance.label = dgcustomfield.label; + localdigicertfieldinstance.is_active = dgcustomfield.is_active; + localdigicertfieldinstance.data_type = dgcustomfield.data_type; + localdigicertfieldinstance.is_required = dgcustomfield.is_required; + + foreach (var kffieldeq in kfcustomfields) + if (dgcustomfield.label == kffieldeq.DigicertFieldName) + localdigicertfieldinstance.kf_field_name = kffieldeq.DigicertFieldName; - if (!fullcustomdgfieldlist.Any(p => p.label == localdigicertfieldinstance.label)) - { fullcustomdgfieldlist.Add(localdigicertfieldinstance); - newcustomfieldsfordg.Add(localdigicertfieldinstance); } - } - //Add fields that don't exist on DigiCert to Digicert - _logger.Trace("Adding following fields to DigiCert: {0}", JsonConvert.SerializeObject(newcustomfieldsfordg)); - foreach (var newdgfield in newcustomfieldsfordg) - { - var digicertapilocation = "https://www.digicert.com/services/v2/account/metadata"; - var digicertnewfieldsclient = new RestClient(); - var digicertnewfieldsrequest = new RestRequest(digicertapilocation); - digicertnewfieldsrequest.AddHeader("Accept", "application/json"); - digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikeytopperm); - var serializedsyncfield = JsonConvert.SerializeObject(newdgfield); - digicertnewfieldsrequest.AddParameter("application/json", serializedsyncfield, - ParameterType.RequestBody); - var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); - } + //This covers all of the new fields on Keyfactors side, including new ones - needs to have digicert ids for the new ones + foreach (var kfcustomfield in kfcustomfields) + { + var localdigicertfieldinstance = new DigicertCustomFieldInstance(); + localdigicertfieldinstance.label = kfcustomfield.DigicertFieldName; + localdigicertfieldinstance.is_active = true; + localdigicertfieldinstance.kf_field_name = kfcustomfield.KeyfactorMetadataFieldName; + if (kfcustomfield.KeyfactorDataType == "String") + localdigicertfieldinstance.data_type = "text"; + else if (kfcustomfield.KeyfactorDataType == "Int") + localdigicertfieldinstance.data_type = "int"; + else + localdigicertfieldinstance.data_type = "anything"; + localdigicertfieldinstance.is_required = false; + + if (!fullcustomdgfieldlist.Any(p => p.label == localdigicertfieldinstance.label)) + { + fullcustomdgfieldlist.Add(localdigicertfieldinstance); + newcustomfieldsfordg.Add(localdigicertfieldinstance); + } + } - // Grabbing the list again from digicert, populating ids for new ones - //Getting list of custom metadata fields on DigiCert - var updatedmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey, importdeactivated); - foreach (var subitem in updatedmetadatafieldlist) - foreach (var fulllistitem in fullcustomdgfieldlist) - if (subitem.label == fulllistitem.label) - fulllistitem.id = subitem.id; + //Add fields that don't exist on DigiCert to Digicert + _logger.Trace("Adding following fields to DigiCert: {0}", + JsonConvert.SerializeObject(newcustomfieldsfordg)); + foreach (var newdgfield in newcustomfieldsfordg) + { + var digicertapilocation = "https://www.digicert.com/services/v2/account/metadata"; + var digicertnewfieldsclient = new RestClient(); + var digicertnewfieldsrequest = new RestRequest(digicertapilocation); + digicertnewfieldsrequest.AddHeader("Accept", "application/json"); + digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikeytopperm); + var serializedsyncfield = JsonConvert.SerializeObject(newdgfield); + digicertnewfieldsrequest.AddParameter("application/json", serializedsyncfield, + ParameterType.RequestBody); + var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); + } - var totalcertsprocessed = 0; - var numcertsdatauploaded = 0; - // Pushing the data to DigiCert - var certlist2 = JsonConvert.DeserializeObject(rawresponse, - new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - foreach (var cert in certlist2) - { - Dictionary kfstoredmetadata = cert["Metadata"].ToObject>(); + // Grabbing the list again from digicert, populating ids for new ones + //Getting list of custom metadata fields on DigiCert + var updatedmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey, importdeactivated); + foreach (var subitem in updatedmetadatafieldlist) + foreach (var fulllistitem in fullcustomdgfieldlist) + if (subitem.label == fulllistitem.label) + fulllistitem.id = subitem.id; - var certhascustomfields = false; - foreach (var checkfield in fullcustomdgfieldlist) - if (kfstoredmetadata.ContainsKey(checkfield.kf_field_name)) - certhascustomfields = true; - if (certhascustomfields) + // Pushing the data to DigiCert + var certlist2 = JsonConvert.DeserializeObject(rawresponse, + new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + foreach (var cert in certlist2) { - var kfserialnumber = cert["SerialNumber"].ToString(); + Dictionary kfstoredmetadata = + cert["Metadata"].ToObject>(); - var digicertnewlookupurl = "https://www.digicert.com/services/v2/order/certificate" + - "?filters[serial_number]=" + kfserialnumber; + var certhascustomfields = false; + foreach (var checkfield in fullcustomdgfieldlist) + if (kfstoredmetadata.ContainsKey(checkfield.kf_field_name)) + certhascustomfields = true; - var newbodytemplate = new RootDigicertLookup(); - var newsearchcriterioninstance = new SearchCriterion(); - newbodytemplate.searchCriteriaList.Add(newsearchcriterioninstance); - var lookupnewrequest = new RestRequest(digicertnewlookupurl); - lookupnewrequest.AddHeader("Content-Type", "application/json"); - lookupnewrequest.AddHeader("X-DC-DEVKEY", digicertapikey); - var digicertnewlookupresponse = client.Execute(lookupnewrequest); - var newparseddigicertresponse = - JsonConvert.DeserializeObject(digicertnewlookupresponse.Content); + if (certhascustomfields) + { + var kfserialnumber = cert["SerialNumber"].ToString(); + var digicertnewlookupurl = "https://www.digicert.com/services/v2/order/certificate" + + "?filters[serial_number]=" + kfserialnumber; - if (newparseddigicertresponse["page"]["total"] != 0) - { - var newflatteneddigicertinstance = newparseddigicertresponse["orders"][0]; - var orderid = newflatteneddigicertinstance["id"].ToString(); + var newbodytemplate = new RootDigicertLookup(); + var newsearchcriterioninstance = new SearchCriterion(); + newbodytemplate.searchCriteriaList.Add(newsearchcriterioninstance); + var lookupnewrequest = new RestRequest(digicertnewlookupurl); + lookupnewrequest.AddHeader("Content-Type", "application/json"); + lookupnewrequest.AddHeader("X-DC-DEVKEY", digicertapikey); + var digicertnewlookupresponse = client.Execute(lookupnewrequest); + var newparseddigicertresponse = + JsonConvert.DeserializeObject(digicertnewlookupresponse.Content); - var digicertmetadataupdateapilocation = - "https://www.digicert.com/services/v2/order/certificate/" + orderid + "/custom-field"; - var digicertnewfieldsclient = new RestClient(); - var digicertnewfieldsrequest = new RestRequest(digicertmetadataupdateapilocation); - digicertnewfieldsrequest.AddHeader("Accept", "application/json"); - digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikey); - foreach (var newfield in fullcustomdgfieldlist) + if (newparseddigicertresponse["page"]["total"] != 0) { - var keyfactorfieldname = ""; - var datauploaded = false; - //Lookup the keyfactor name for digicert fields - foreach (var sublookup in kfcustomfields) - if (sublookup.DigicertFieldName == newfield.label) - { - var metadatapayload = new Dictionary(); - metadatapayload["metadata_id"] = newfield.id.ToString(); - if (kfstoredmetadata.ContainsKey(sublookup.KeyfactorMetadataFieldName)) + var newflatteneddigicertinstance = newparseddigicertresponse["orders"][0]; + var orderid = newflatteneddigicertinstance["id"].ToString(); + + var digicertmetadataupdateapilocation = + "https://www.digicert.com/services/v2/order/certificate/" + orderid + "/custom-field"; + var digicertnewfieldsclient = new RestClient(); + var digicertnewfieldsrequest = new RestRequest(digicertmetadataupdateapilocation); + digicertnewfieldsrequest.AddHeader("Accept", "application/json"); + digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikey); + + foreach (var newfield in fullcustomdgfieldlist) + { + var keyfactorfieldname = ""; + var datauploaded = false; + //Lookup the keyfactor name for digicert fields + foreach (var sublookup in kfcustomfields) + if (sublookup.DigicertFieldName == newfield.label) { - metadatapayload["value"] = - kfstoredmetadata[sublookup.KeyfactorMetadataFieldName]; - var newserializedsyncfield = JsonConvert.SerializeObject(metadatapayload); - digicertnewfieldsrequest.AddParameter("application/json", - newserializedsyncfield, ParameterType.RequestBody); - var digicertresponsenewfields = - digicertnewfieldsclient.Post(digicertnewfieldsrequest); - datauploaded = true; + var metadatapayload = new Dictionary(); + metadatapayload["metadata_id"] = newfield.id.ToString(); + if (kfstoredmetadata.ContainsKey(sublookup.KeyfactorMetadataFieldName)) + { + metadatapayload["value"] = + kfstoredmetadata[sublookup.KeyfactorMetadataFieldName]; + var newserializedsyncfield = JsonConvert.SerializeObject(metadatapayload); + digicertnewfieldsrequest.AddParameter("application/json", + newserializedsyncfield, ParameterType.RequestBody); + var digicertresponsenewfields = + digicertnewfieldsclient.Post(digicertnewfieldsrequest); + datauploaded = true; + } } - } - } + } - numcertsdatauploaded += 1; + numcertsdatauploaded += 1; + } } + + totalcertsprocessed += 1; } - totalcertsprocessed += 1; - } - Console.WriteLine( - $"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); - Console.WriteLine($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); - _logger.Debug( - $"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); - _logger.Debug($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); + // Update the count of items downloaded so far + certsdownloaded += batchsize; + + // Check if all items have been downloaded + if (certlist.Count == 0) + { + Console.WriteLine( + $"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); + Console.WriteLine($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); + _logger.Debug( + $"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); + _logger.Debug($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); ; + + break; + } + } } // Syncing Data from DigiCert TO Keyfactor if (config_mode == "dctokf") { - Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - _logger.Debug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - //Each cert that is DigiCert in origin in Keyfactor is looked up on DigiCert via serial number, - //and the metadata contents from those fields are stored. - var digicertlookupclient = new RestClient(); - var digicertcertificates = new List(); - foreach (var certinstance in certlist) + // Initialize variable to keep track of items downloaded so far + int certsdownloaded = 0; + var certcounttracker = 0; + for (int batchnum = 0; batchnum < numberOfBatches; batchnum++) { - var digicertlookupurl = "https://www.digicert.com/services/v2/order/certificate/"; - - var bodytemplate = new RootDigicertLookup(); - var searchcriterioninstance = new SearchCriterion(); - bodytemplate.searchCriteriaList.Add(searchcriterioninstance); - - digicertlookupurl = digicertlookupurl + certinstance.SerialNumber; - var lookuprequest = new RestRequest(digicertlookupurl); - lookuprequest.AddHeader("Content-Type", "application/json"); - lookuprequest.AddHeader("X-DC-DEVKEY", digicertapikey); - var digicertlookupresponse = client.Execute(lookuprequest); - var certcontent = JsonConvert.DeserializeObject(digicertlookupresponse.Content, - new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - - if (certcontent["certificate"] != null) + // Check if reaching the arbitrary limit + if (certsdownloaded + batchsize > returnlimitint) { - digicertcertificates.Add(certcontent); - _logger.Trace("Pulled and storing following cert from digicert: {0}", - digicertlookupresponse.Content); + Console.WriteLine($"Stopped downloading at the configured limit of {returnlimitint} items."); + _logger.Debug($"Stopped downloading at the configured limit of {returnlimitint} items."); + break; } - Console.WriteLine("Pulled DigiCert matching DigiCert cert data."); - _logger.Debug("Pulled DigiCert matching DigiCert cert data."); + // Download the items in this batch + Console.WriteLine($"Downloading batch {batchnum + 1}..."); - var certcounttracker = 0; - foreach (var digicertcertinstance in digicertcertificates) - { - var finalsyncclient = new RestClient(); - finalsyncclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); - var finalsyncurl = keyfactorapilocation + "Certificates/Metadata"; - //Find matching certificate via Keyfactor ID - var test = digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper(); - var query = from kfcertlocal in certlist - where kfcertlocal.SerialNumber == - digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper() - select kfcertlocal; - var certificateid = query.FirstOrDefault().Id; - } + var digicertlookup = keyfactorapilocation + "Certificates?pq.queryString=IssuerDN%20-contains%20%22" + + digicertIssuerQueryterm + "%22&pq.returnLimit=" + batchsize.ToString() + + "&includeMetadata=true" + "&pq.pageReturned=" + batchnum.ToString(); + var request = new RestRequest(digicertlookup); + request.AddHeader("Accept", "application/json"); + request.AddHeader("x-keyfactor-api-version", "1"); + request.AddHeader("x-keyfactor-requested-with", "APIClient"); + var response = client.Execute(request); + var rawresponse = response.Content; + var certlist = JsonConvert.DeserializeObject>(rawresponse, + new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + Console.WriteLine("Got DigiCert issued certs from keyfactor"); + _logger.Debug("Got DigiCert issued certs from keyfactor"); + + //Each cert that is DigiCert in origin in Keyfactor is looked up on DigiCert via serial number, + //and the metadata contents from those fields are stored. + var digicertlookupclient = new RestClient(); + var digicertcertificates = new List(); + foreach (var certinstance in certlist) + { + var digicertlookupurl = "https://www.digicert.com/services/v2/order/certificate/"; + var bodytemplate = new RootDigicertLookup(); + var searchcriterioninstance = new SearchCriterion(); + bodytemplate.searchCriteriaList.Add(searchcriterioninstance); - if (digicertcertinstance["custom_fields"] != null) - // Getting custom metadata field values - foreach (var metadatafieldinstance in digicertcertinstance["custom_fields"]) - if (importallcustomdigicertfields) + digicertlookupurl = digicertlookupurl + certinstance.SerialNumber; + var lookuprequest = new RestRequest(digicertlookupurl); + lookuprequest.AddHeader("Content-Type", "application/json"); + lookuprequest.AddHeader("X-DC-DEVKEY", digicertapikey); + var digicertlookupresponse = client.Execute(lookuprequest); + var certcontent = JsonConvert.DeserializeObject(digicertlookupresponse.Content, + new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + + if (certcontent["certificate"] != null) { - // Using autoimport and thus using autorename - var metadatanamefield = ReplaceAllWhiteSpaces(metadatafieldinstance["label"].ToString(), - replacementcharacter); - payloadforkf.Metadata[metadatanamefield] = metadatafieldinstance["value"]; + digicertcertificates.Add(certcontent); + _logger.Trace("Pulled and storing following cert from digicert: {0}", + digicertlookupresponse.Content); } - } - - } - - var totalcertsprocessed = 0; - var numcertsdatauploaded = 0; - - // Pushing the data to DigiCert - var certlist2 = JsonConvert.DeserializeObject(rawresponse, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - foreach (var cert in certlist2) - { - - Dictionary kfstoredmetadata = cert["Metadata"].ToObject>(); - - bool certhascustomfields = false; - foreach (var checkfield in fullcustomdgfieldlist) - { - if (kfstoredmetadata.ContainsKey(checkfield.kf_field_name)) + else { - certhascustomfields = true; + _logger.Trace("Failed to retrieve cert {0} from Digicert", digicertlookupresponse.Content); } } - if (certhascustomfields){ - var kfserialnumber = cert["SerialNumber"].ToString(); + Console.WriteLine("Pulled DigiCert matching DigiCert cert data."); + _logger.Debug("Pulled DigiCert matching DigiCert cert data."); - var digicertnewlookupurl = "https://www.digicert.com/services/v2/order/certificate" + "?filters[serial_number]=" + kfserialnumber; + foreach (var digicertcertinstance in digicertcertificates) + { + var finalsyncclient = new RestClient(); + finalsyncclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); + var finalsyncurl = keyfactorapilocation + "Certificates/Metadata"; + //Find matching certificate via Keyfactor ID + var test = digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper(); + var query = from kfcertlocal in certlist + where kfcertlocal.SerialNumber == + digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper() + select kfcertlocal; + var certificateid = query.FirstOrDefault().Id; + + + var payloadforkf = new KeyfactorMetadataQuery(); + payloadforkf.Id = certificateid; + + if (digicertcertinstance["custom_fields"] != null) + // Getting custom metadata field values + foreach (var metadatafieldinstance in digicertcertinstance["custom_fields"]) + if (importallcustomdigicertfields) + { + // Using autoimport and thus using autorename + var metadatanamefield = ReplaceAllBannedCharacters(metadatafieldinstance["label"].ToString(), + allBannedChars); + payloadforkf.Metadata[metadatanamefield] = metadatafieldinstance["value"]; + } + else + { + //Using custom names + var metadatanamequery = from customfieldinstance in kfcustomfields + where customfieldinstance.DigicertFieldName == + metadatafieldinstance["label"] + select customfieldinstance; + if (metadatanamequery.FirstOrDefault() != null) + payloadforkf.Metadata[metadatanamequery.FirstOrDefault().DigicertFieldName] = + metadatafieldinstance["value"]; + } - var newbodytemplate = new RootDigicertLookup(); - var newsearchcriterioninstance = new SearchCriterion(); - newbodytemplate.searchCriteriaList.Add(newsearchcriterioninstance); - var lookupnewrequest = new RestRequest(digicertnewlookupurl); - lookupnewrequest.AddHeader("Content-Type", "application/json"); - lookupnewrequest.AddHeader("X-DC-DEVKEY", digicertapikey); - var digicertnewlookupresponse = client.Execute(lookupnewrequest); - var newparseddigicertresponse = JsonConvert.DeserializeObject(digicertnewlookupresponse.Content); + var flattenedcert = Flatten(digicertcertinstance); + //Getting manually selected metadata field values (not custom in DigiCert) + foreach (var manualinstance in kfmanualfields) + if (flattenedcert[manualinstance.DigicertFieldName] != null) + payloadforkf.Metadata[manualinstance.KeyfactorMetadataFieldName] = + flattenedcert[manualinstance.DigicertFieldName].ToString(); + //Sending the payload off to Keyfactor for the update + var finalsyncreq = new RestRequest(finalsyncurl); + finalsyncreq.AddHeader("Content-Type", "application/json"); + finalsyncreq.AddHeader("x-keyfactor-api-version", "1"); + finalsyncreq.AddHeader("x-keyfactor-requested-with", "APIClient"); + var serializedsyncfield = JsonConvert.SerializeObject(payloadforkf); + _logger.Trace("Sending Metadata update to KF for cert ID {0}, metadata update: {1}", + payloadforkf.Id.ToString(), serializedsyncfield); + + finalsyncreq.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); + finalsyncclient.Put(finalsyncreq); + ++certcounttracker; + } + - if (newparseddigicertresponse["page"]["total"] != 0) - { - //Using custom names - var metadatanamequery = from customfieldinstance in kfcustomfields - where customfieldinstance.DigicertFieldName == - metadatafieldinstance["label"] - select customfieldinstance; - if (metadatanamequery.FirstOrDefault() != null) - payloadforkf.Metadata[metadatanamequery.FirstOrDefault().DigicertFieldName] = - metadatafieldinstance["value"]; - } - } - totalcertsprocessed += 1; - } - Console.WriteLine($"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); - Console.WriteLine($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); - logger.LogDebug($"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); - logger.LogDebug($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); + // Update the count of items downloaded so far + certsdownloaded += batchsize; - } + // Check if all items have been downloaded + if (certlist.Count == 0) + { + Console.WriteLine( + $"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); + _logger.Debug( + $"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); - var flattenedcert = Flatten(digicertcertinstance); - //Getting manually selected metadata field values (not custom in DigiCert) - foreach (var manualinstance in kfmanualfields) - if (flattenedcert[manualinstance.DigicertFieldName] != null) - payloadforkf.Metadata[manualinstance.KeyfactorMetadataFieldName] = - flattenedcert[manualinstance.DigicertFieldName].ToString(); - //Sending the payload off to Keyfactor for the update - var finalsyncreq = new RestRequest(finalsyncurl); - finalsyncreq.AddHeader("Content-Type", "application/json"); - finalsyncreq.AddHeader("x-keyfactor-api-version", "1"); - finalsyncreq.AddHeader("x-keyfactor-requested-with", "APIClient"); - var serializedsyncfield = JsonConvert.SerializeObject(payloadforkf); - _logger.Trace("Sending Metadata update to KF for cert ID {0}, metadata update: {1}", - payloadforkf.Id.ToString(), serializedsyncfield); - - finalsyncreq.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); - finalsyncclient.Put(finalsyncreq); - ++certcounttracker; + break; + } } - - Console.WriteLine( - $"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); - _logger.Debug( - $"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); } + + Environment.Exit(0); } -} \ No newline at end of file +} diff --git a/digicert-metadata-sync/Models/CharDBItem.cs b/digicert-metadata-sync/Models/CharDBItem.cs new file mode 100644 index 0000000..63e12c9 --- /dev/null +++ b/digicert-metadata-sync/Models/CharDBItem.cs @@ -0,0 +1,26 @@ +// Copyright 2021 Keyfactor +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace DigicertMetadataSync; + +partial class DigicertSync +{ + + public class CharDBItem + { + public string character; + public string replacementcharacter; + } + +} \ No newline at end of file diff --git a/digicert-metadata-sync/Models/InternalClasses.cs b/digicert-metadata-sync/Models/InternalClasses.cs index 1cf7082..5c0d3eb 100644 --- a/digicert-metadata-sync/Models/InternalClasses.cs +++ b/digicert-metadata-sync/Models/InternalClasses.cs @@ -34,6 +34,7 @@ public class ReadInMetadataField public string KeyfactorDataType { get; set; } = "string"; public string KeyfactorHint { get; set; } = "None."; public string KeyfactorAllowAPI { get; set; } = "True"; + public string FieldType { get; set; } = "manual/custom"; } public class KeyfactorMetadataInstanceSendoff diff --git a/digicert-metadata-sync/replacechar.json b/digicert-metadata-sync/replacechar.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/digicert-metadata-sync/replacechar.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/readme_source.md b/readme_source.md index 662d007..e7d257b 100644 --- a/readme_source.md +++ b/readme_source.md @@ -34,11 +34,17 @@ This should include the Keyfactor API endpoint, of the format https://domain.com This should include the common prefix all DigiCert certs have in your Keyfactor instance. For example, "DigiCert" - ImportAllCustomDigicertFields This setting enables the tool to import all of the custom metadata fields included in DigiCert and sync all of their data. -- ReplaceDigicertWhiteSpaceCharacterInName -In case the ImportAllCustomDigicertFields setting is used, this is necessary to for metadata field label conversion. DigiCert supports spaces in labels and Keyfactor does not, so this replaces the spaces in the name with your character sequence of choice. + +During the first run, the tool will scan the custom fields it will be importing for characters that are not supported in Keyfactor Metadata field names. +Each unsupported character will be shown in a file named "replacechar.json" and its replacement can be selected. If the values in the file are not populated, the tool will not run a second time. - ImportDataForDeactivatedDigiCertFields If this is enabled, custom metadata fields that were deactivated in DigiCert will also be synced, and the data stored in these fields in certificates will be too. +### replacechar.json settings +This file is populated during the first run of the tool if the ImportAllCustomDigicertFields setting is toggled. +The only text that needs replacing is shown as "null", and can be filled with any alphanumeric string. The "_" and "-" characters are also supported. + + ### manualfields.json settings This file is used to specify which metadata fields should be synced up.