From 3c7663b1d4e207610f1fa21e8c19e137483d6e50 Mon Sep 17 00:00:00 2001 From: Mark Kachkaev <37276742+mkachk@users.noreply.github.com> Date: Fri, 19 May 2023 11:02:28 -0400 Subject: [PATCH 01/12] Version 2.0 - added sync from KF to DC. (#12) * Version 2.0 - added sync from KF to DC. --- CHANGELOG.md | 8 +- README.md | 16 +- .../AddFieldsToKeyfactor.cs | 8 +- digicert-metadata-sync/App.config | 5 +- .../GrabCustomFieldsFromDigiCert.cs | 44 ++ digicert-metadata-sync/Helpers.cs | 8 + digicert-metadata-sync/MetadataSync.cs | 387 +++++++++++++----- .../Models/DigicertCustomField.cs | 36 ++ .../Models/KeyfactorCertInstance.cs | 1 + readme_source.md | 16 +- 10 files changed, 414 insertions(+), 115 deletions(-) create mode 100644 digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs create mode 100644 digicert-metadata-sync/Models/DigicertCustomField.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 685852d..8247963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +Version 2.0 + +- Added ability to sync custom fields from Keyfactor to DigiCert. +- Tool now requires command line argument to specify sync direction: "dctokf" for DigiCert to Keyfactor and "kftodc" for Keyfactor to DigiCert. +- New DigiCert API Key with restrictions set to "None" in DigiCert config required to perform sync from Keyfactor to Digicert. + Version 1.0 -Initial Release +- Initial Release diff --git a/README.md b/README.md index 9b40160..7465f0e 100644 --- a/README.md +++ b/README.md @@ -20,19 +20,30 @@ ___ + + ## Overview -This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. +This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. The latest update allows for syncronization of custom field contents from Keyfactor to DigiCert. New fields are created in Keyfactor and DigiCert to accomodate for this. ## Installation and Usage The tool comes as a Windows executable. The tool performs synchronization each time its run. For the tool to run automatically, it needs to be added as a scheduled process using Windows. The advised interval for running it is once per week. The files App.config and manualfields.json need to be present in the same directory as the tool for it to run correctly. The specific location from which the tool is ran does not matter, but it needs to have access to both the Keyfactor API endpoint as well as Digicert, and appropriate permissions for access to the configuration files. An explanation for the settings found in these files is given below. +## Command Line Arguments +One of these two arguments needs to be used for the tool to run. +- "kftodc" +Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. Example: ```.\DigicertMetadataSync.exe kftodc``` +- "dctokf" +Syncronizes the contents of both custom and non-custom fields from DigiCert to Keyfactor. The fields are listed in manualfields.json, and are created if necessary. +Example: ```.\DigicertMetadataSync.exe dctokf``` ## Settings The settings currently present in these files are shown as an example and need to be configured for your specific situation. ### app.config settings - DigicertAPIKey -Standard DigiCert API access key +Standard DigiCert API access key. +- DigicertAPIKeyTopPerm +DigiCert API access key with restrictions set to "None" - required for sync from Keyfactor to DigiCert. - KeyfactorDomainAndUser Same credential as used when logging into Keyfactor Command. A different set of credentials can be used provided they have adequate access permissions. - KeyfactorPassword @@ -76,3 +87,4 @@ String to be input into Keyfactor as the metadata field hint. - KeyfactorAllowAPI Allows API management of this metadata field in Keyfactor. Should be set to true for continuous synchronization with this tool. + diff --git a/digicert-metadata-sync/AddFieldsToKeyfactor.cs b/digicert-metadata-sync/AddFieldsToKeyfactor.cs index 33289d3..363da96 100644 --- a/digicert-metadata-sync/AddFieldsToKeyfactor.cs +++ b/digicert-metadata-sync/AddFieldsToKeyfactor.cs @@ -22,7 +22,7 @@ namespace DigicertMetadataSync; // It will only add new fields. partial class DigicertSync { - public static int AddFieldsToKeyfactor(List inputlist, + public static Tuple> AddFieldsToKeyfactor(List inputlist, List existingmetadatalist, bool noexistingfields, string keyfactorusername, string keyfactorpassword, string keyfactorapilocation) { @@ -30,6 +30,7 @@ public static int AddFieldsToKeyfactor(List in var addfieldsclient = new RestClient(); addfieldsclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); int totalnumberadded = 0; + List newfields = new List(); foreach (var metadatainstance in inputlist) { if (noexistingfields == false) @@ -37,6 +38,7 @@ public static int AddFieldsToKeyfactor(List in var fieldquery = from existingmetadatainstance in existingmetadatalist where existingmetadatainstance.Name == metadatainstance.Name select existingmetadatainstance; + // If field does not exist in Keyfactor, add it. if (!fieldquery.Any()) { var addfieldrequest = new RestRequest(addfieldstokeyfactorurl); @@ -50,6 +52,7 @@ public static int AddFieldsToKeyfactor(List in try { metadataresponse = addfieldsclient.Post(addfieldrequest); + newfields.Add(metadatainstance.Name); ++totalnumberadded; } catch (HttpRequestException e) @@ -90,7 +93,8 @@ public static int AddFieldsToKeyfactor(List in } } } + Tuple> returnvals = new Tuple>(totalnumberadded, newfields); - return totalnumberadded; + return returnvals; } } \ No newline at end of file diff --git a/digicert-metadata-sync/App.config b/digicert-metadata-sync/App.config index 40e5a24..3d97fda 100644 --- a/digicert-metadata-sync/App.config +++ b/digicert-metadata-sync/App.config @@ -2,10 +2,11 @@ + - - + + diff --git a/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs b/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs new file mode 100644 index 0000000..834097e --- /dev/null +++ b/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs @@ -0,0 +1,44 @@ +// Copyright 2021 Keyfactor +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Keyfactor.Logging; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using RestSharp; +using RestSharp.Authenticators; + +namespace DigicertMetadataSync; + +// This fuction adds the fields to keyfactor. +// It will only add new fields. +partial class DigicertSync +{ + public static List GrabCustomFieldsFromDigiCert(string apikey) + { + ILogger logger = LogHandler.GetClassLogger(); + var digicertclient = new RestClient(); + var customfieldsretrieval = "https://www.digicert.com/services/v2/account/metadata"; + var digicertrequest = new RestRequest(customfieldsretrieval); + digicertrequest.AddHeader("Accept", "application/json"); + digicertrequest.AddHeader("X-DC-DEVKEY", apikey); + var digicertresponse = digicertclient.Execute(digicertrequest); + var trimmeddigicertresponse = digicertresponse.Content.Remove(0, 12); + int lengthofresponse = trimmeddigicertresponse.Length; + trimmeddigicertresponse = trimmeddigicertresponse.Remove(lengthofresponse - 1, 1); + var fieldlist = JsonConvert.DeserializeObject>(trimmeddigicertresponse); + Console.WriteLine("Obtained custom fields from DigiCert."); + logger.LogDebug("Obtained custom fields from DigiCert."); + return fieldlist; + } +} \ No newline at end of file diff --git a/digicert-metadata-sync/Helpers.cs b/digicert-metadata-sync/Helpers.cs index 2c5eaee..b44c192 100644 --- a/digicert-metadata-sync/Helpers.cs +++ b/digicert-metadata-sync/Helpers.cs @@ -70,6 +70,14 @@ public static string ReplaceAllWhiteSpaces(string str, string replacement) return Regex.Replace(str, @"\s+", "_-_"); } + public static bool CheckMode(string mode) + { + if ((mode == "kftodc") || (mode == "dctokf")){ + return true; + } + return false; + } + private static List convertlisttokf(List inputlist, string replacementcharacter) { diff --git a/digicert-metadata-sync/MetadataSync.cs b/digicert-metadata-sync/MetadataSync.cs index 4b0baa7..1e36077 100644 --- a/digicert-metadata-sync/MetadataSync.cs +++ b/digicert-metadata-sync/MetadataSync.cs @@ -25,6 +25,9 @@ using Keyfactor; using Microsoft.Extensions.Logging; using Keyfactor.Logging; +using static DigicertMetadataSync.DigicertSync; +using System.Collections.Immutable; +using System.Runtime.CompilerServices; namespace DigicertMetadataSync { @@ -36,19 +39,27 @@ public static void Main(string[] args) ILogger logger = LogHandler.GetClassLogger(); logger.LogDebug("Start sync"); var digicertapikey = System.Configuration.ConfigurationManager.AppSettings.Get("DigicertAPIKey"); + var digicertapikeytopperm = System.Configuration.ConfigurationManager.AppSettings.Get("DigicertAPIKeyTopPerm"); var keyfactorusername = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorDomainAndUser"); var keyfactorpassword = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorPassword"); var replacementcharacter = System.Configuration.ConfigurationManager.AppSettings.Get("ReplaceDigicertWhiteSpaceCharacterInName"); var importallcustomdigicertfields = Convert.ToBoolean(System.Configuration.ConfigurationManager.AppSettings.Get("ImportAllCustomDigicertFields")); - //Get list of all digicert certs from keyfactor based on query that contains digicert as issuer. + var config_mode = args[0]; + if (CheckMode(config_mode) == false) + { + logger.LogDebug("Inappropriate configuration mode. Check your command line arguments."); + throw new Exception("Inappropriate configuration mode. Check your command line arguments."); + } + + //Get list of all DigiCert certs from Keyfactor based on query that contains DigiCert as issuer. var returnlimit = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorCertSearchReturnLimit").ToString(); var keyfactorapilocation = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorAPIEndpoint").ToString(); var digicertIssuerQueryterm = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorDigicertIssuedCertQueryTerm").ToString(); logger.LogDebug($"Loaded config. Processing with a Keyfactor Query Return Limit of {returnlimit.ToString()}"); var digicertlookup = keyfactorapilocation + "Certificates?pq.queryString=IssuerDN%20-contains%20%22" - + digicertIssuerQueryterm + "%22&pq.returnLimit=" + returnlimit; + + digicertIssuerQueryterm + "%22&pq.returnLimit=" + returnlimit + "&includeMetadata=true"; var client = new RestClient(); client.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); var request = new RestRequest(digicertlookup); @@ -58,8 +69,8 @@ public static void Main(string[] args) var response = client.Execute(request); var rawresponse = response.Content; var certlist = JsonConvert.DeserializeObject>(rawresponse, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - Console.WriteLine("Got digicert issued certs from keyfactor"); - logger.LogDebug("Got digicert issued certs from keyfactor"); + Console.WriteLine("Got DigiCert issued certs from keyfactor"); + logger.LogDebug("Got DigiCert issued certs from keyfactor"); //Getting list of custom metadata fields from Keyfactor var getmetadalistkf = keyfactorapilocation + "MetadataFields"; @@ -75,28 +86,17 @@ public static void Main(string[] args) Console.WriteLine("Got list of custom fields from Keyfactor."); logger.LogDebug("Got list of custom fields from Keyfactor."); - //Getting list of custom metadata fields on Digicert - var digicertclient = new RestClient(); - var customfieldsretrieval = "https://www.digicert.com/services/v2/account/metadata"; - var digicertrequest = new RestRequest(customfieldsretrieval); - digicertrequest.AddHeader("Accept", "application/json"); - digicertrequest.AddHeader("X-DC-DEVKEY", digicertapikey); - var digicertresponse = client.Execute(digicertrequest); - var trimmeddigicertresponse = digicertresponse.Content.Remove(0, 12); - int lengthofresponse = trimmeddigicertresponse.Length; - trimmeddigicertresponse = trimmeddigicertresponse.Remove(lengthofresponse - 1, 1); - var customdigicertmetadatafieldlist = JsonConvert.DeserializeObject>(trimmeddigicertresponse); - Console.WriteLine("Obtained custom fields from digicert."); - logger.LogDebug("Obtained custom fields from digicert."); - - //Convert digicert custom fields to keyfactor appropriate ones + //Getting list of custom metadata fields on DigiCert + var customdigicertmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey); + + //Convert DigiCert custom fields to Keyfactor appropriate ones //This depends on whether the setting to import all fields was enabled or not var config = new ConfigurationBuilder().SetBasePath(AppDomain.CurrentDomain.BaseDirectory).AddJsonFile("manualfields.json").Build(); var kfcustomfields = new List(); if (importallcustomdigicertfields == true) { - //This imports all the custom fields based on the list of metadata from Digicert and does autofill + //This imports all the custom fields based on the list of metadata from DigiCert and does autofill for (int i = 0; i < customdigicertmetadatafieldlist.Count; i++) { var localkffieldinstance = new ReadInMetadataField(); @@ -112,10 +112,10 @@ public static void Main(string[] args) if (customdigicertmetadatafieldlist[i].label != null) { /* - NOTICE: KEYFACTOR DOES NOT SUPPORT SPACES IN METADATA FIELD NAMES. + NOTICE: KEYFACTOR DOES NOT SUPPORT SPACES IN METADATA FIELD NAMES. WHITESPACE MUST BE REMOVED FROM THE NAME. CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. - */ + */ localkffieldinstance.DigicertFieldName = customdigicertmetadatafieldlist[i].label; localkffieldinstance.KeyfactorMetadataFieldName = ReplaceAllWhiteSpaces(customdigicertmetadatafieldlist[i].label, replacementcharacter); } @@ -136,7 +136,7 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. kfcustomfields = config.GetSection(customfieldslst).Get>(); } - //Adding metadata fields for the ID and the email of the requester from digicert. + //Adding metadata fields for the ID and the email of the requester from DigiCert. List kfmanualfields = new List(); var manualfieldslist = "ManualFields"; kfmanualfields = config.GetSection(manualfieldslist).Get>(); @@ -161,7 +161,7 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. noexistingfields = false; } Console.WriteLine("Pulled existing metadata fields from keyfactor."); - logger.LogDebug("Pulled existing metadata fields from keyfactor."); + logger.LogDebug("Pulled existing metadata fields from Keyfactor."); // Converting the read in fields into sendable lists var convertedmanualfields = convertlisttokf(kfmanualfields, replacementcharacter); var convertedcustomfields = convertlisttokf(kfcustomfields, replacementcharacter); @@ -169,119 +169,294 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. int totalfieldsadded = 0; //If all the fields are absent from Keyfactor, the fields are added. - totalfieldsadded += AddFieldsToKeyfactor(convertedmanualfields, existingmetadatalist, noexistingfields, keyfactorusername, keyfactorpassword, keyfactorapilocation); - totalfieldsadded += AddFieldsToKeyfactor(convertedcustomfields, existingmetadatalist, noexistingfields, keyfactorusername, keyfactorpassword, keyfactorapilocation); + var manualresult = AddFieldsToKeyfactor(convertedmanualfields, existingmetadatalist, noexistingfields, keyfactorusername, keyfactorpassword, keyfactorapilocation); + var customresult = AddFieldsToKeyfactor(convertedcustomfields, existingmetadatalist, noexistingfields, keyfactorusername, keyfactorpassword, keyfactorapilocation); + totalfieldsadded += manualresult.Item1; + totalfieldsadded += customresult.Item1; - Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - logger.LogDebug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - //Each cert that is digicert in origin in keyfactor is looked up on Digicert via serial number, - //and the metadata contents from those fields are stored. - var digicertlookupclient = new RestClient(); - List digicertcertificates = new List(); - foreach (var certinstance in certlist) + var allnewfields = manualresult.Item2.Concat(customresult.Item2).ToList(); + // Syncing Data from Keyfactor TO DigiCert + // Sync from DigiCert to Keyfactor must run at least once prior to this - only runs with custom fields + if (config_mode == "kftodc") { - var digicertlookupurl = "https://www.digicert.com/services/v2/order/certificate/"; - - var bodytemplate = new RootDigicertLookup(); - var searchcriterioninstance = new SearchCriterion(); - bodytemplate.searchCriteriaList.Add(searchcriterioninstance); - - digicertlookupurl = digicertlookupurl + certinstance.SerialNumber; - var lookuprequest = new RestRequest(digicertlookupurl); - lookuprequest.AddHeader("Content-Type", "application/json"); - lookuprequest.AddHeader("X-DC-DEVKEY", digicertapikey); - var digicertlookupresponse = client.Execute(lookuprequest); - var parseddigicertresponse = JsonConvert.DeserializeObject(digicertlookupresponse.Content); - if (parseddigicertresponse.certificate != null) + Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + logger.LogDebug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + + List fullcustomdgfieldlist = new List(); + List newcustomfieldsfordg = new List(); + // Rebuild the list of metadata field names as they are on DigiCerts side. + + // This covers all of the custom fields on Digicerts side + foreach (var dgcustomfield in customdigicertmetadatafieldlist) { - var flatteneddigicertinstance = ClassConverter(parseddigicertresponse); - digicertcertificates.Add(parseddigicertresponse); + DigicertCustomFieldInstance localdigicertfieldinstance = new DigicertCustomFieldInstance(); + + localdigicertfieldinstance.label = dgcustomfield.label; + localdigicertfieldinstance.is_active = dgcustomfield.is_active; + localdigicertfieldinstance.data_type = dgcustomfield.data_type; + localdigicertfieldinstance.is_required = dgcustomfield.is_required; + + foreach (var kffieldeq in kfcustomfields) + { + if (dgcustomfield.label == kffieldeq.DigicertFieldName) + { + localdigicertfieldinstance.kf_field_name = kffieldeq.DigicertFieldName; + } + } + + fullcustomdgfieldlist.Add(localdigicertfieldinstance); } - } - Console.WriteLine("Pulled digicert matching digicert cert data."); - logger.LogDebug("Pulled digicert matching digicert cert data."); - //The metadata for each cert is served back to keyfactor and the fields are updated. + + //This covers all of the new fields on Keyfactors side, including new ones - needs to have digicert ids for the new ones + foreach (var kfcustomfield in kfcustomfields) + { + DigicertCustomFieldInstance localdigicertfieldinstance = new DigicertCustomFieldInstance(); + localdigicertfieldinstance.label = kfcustomfield.DigicertFieldName; + localdigicertfieldinstance.is_active = true; + localdigicertfieldinstance.kf_field_name = kfcustomfield.KeyfactorMetadataFieldName; + if (kfcustomfield.KeyfactorDataType == "String") + { + localdigicertfieldinstance.data_type = "text"; + } + else if (kfcustomfield.KeyfactorDataType == "Int") + { + localdigicertfieldinstance.data_type = "int"; + } + else + { + localdigicertfieldinstance.data_type = "anything"; + } + localdigicertfieldinstance.is_required = false; + + if (!fullcustomdgfieldlist.Any(p => p.label == localdigicertfieldinstance.label)) + { + fullcustomdgfieldlist.Add(localdigicertfieldinstance); + newcustomfieldsfordg.Add(localdigicertfieldinstance); + } + } - int certcounttracker = 0; - foreach (var digicertcertinstance in digicertcertificates) - { - var finalsyncclient = new RestClient(); - finalsyncclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); - var finalsyncurl = keyfactorapilocation+ "Certificates/Metadata"; - //Find matching certificate via Keyfactor ID - var query = from kfcertlocal in certlist - where kfcertlocal.SerialNumber == - digicertcertinstance.certificate.serial_number.ToUpper() - select kfcertlocal; - var certificateid = query.FirstOrDefault().Id; + //Add fields that don't exist on DigiCert to Digicert + foreach (var newdgfield in newcustomfieldsfordg) + { + var digicertapilocation = "https://www.digicert.com/services/v2/account/metadata"; + var digicertnewfieldsclient = new RestClient(); + var digicertnewfieldsrequest = new RestRequest(digicertapilocation); + digicertnewfieldsrequest.AddHeader("Accept", "application/json"); + digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikeytopperm); + var serializedsyncfield = JsonConvert.SerializeObject(newdgfield); + digicertnewfieldsrequest.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); + var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); + } - var payloadforkf = new KeyfactorMetadataQuery(); - payloadforkf.Id = certificateid; - if (digicertcertinstance.custom_fields != null) + // Grabbing the list again from digicert, populating ids for new ones + //Getting list of custom metadata fields on DigiCert + var updatedmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey); + foreach (var subitem in updatedmetadatafieldlist) { - // Getting custom metadata field values - foreach (var metadatafieldinstance in digicertcertinstance.custom_fields) + foreach (var fulllistitem in fullcustomdgfieldlist) { - if (importallcustomdigicertfields == true) + if (subitem.label == fulllistitem.label) { - // Using autoimport and thus using autorename - payloadforkf.Metadata[ReplaceAllWhiteSpaces(metadatafieldinstance.label, replacementcharacter)] = metadatafieldinstance.value; + fulllistitem.id = subitem.id; } - else + } + + } + + var totalcertsprocessed = 0; + var numcertsdatauploaded = 0; + + // Pushing the data to DigiCert + var certlist2 = JsonConvert.DeserializeObject(rawresponse, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + foreach (var cert in certlist2) + { + + Dictionary kfstoredmetadata = cert["Metadata"].ToObject>(); + + bool certhascustomfields = false; + foreach (var checkfield in fullcustomdgfieldlist) + { + if (kfstoredmetadata.ContainsKey(checkfield.kf_field_name)) + { + certhascustomfields = true; + } + } + + if (certhascustomfields){ + var kfserialnumber = cert["SerialNumber"].ToString(); + + var digicertnewlookupurl = "https://www.digicert.com/services/v2/order/certificate" + "?filters[serial_number]=" + kfserialnumber; + + var newbodytemplate = new RootDigicertLookup(); + var newsearchcriterioninstance = new SearchCriterion(); + newbodytemplate.searchCriteriaList.Add(newsearchcriterioninstance); + var lookupnewrequest = new RestRequest(digicertnewlookupurl); + lookupnewrequest.AddHeader("Content-Type", "application/json"); + lookupnewrequest.AddHeader("X-DC-DEVKEY", digicertapikey); + var digicertnewlookupresponse = client.Execute(lookupnewrequest); + var newparseddigicertresponse = JsonConvert.DeserializeObject(digicertnewlookupresponse.Content); + + + if (newparseddigicertresponse["page"]["total"] != 0) { - //Using custom names - var metadatanamequery = from customfieldinstance in kfcustomfields - where customfieldinstance.DigicertFieldName == - metadatafieldinstance.label - select customfieldinstance; - if (metadatanamequery.FirstOrDefault() != null) + var newflatteneddigicertinstance = newparseddigicertresponse["orders"][0]; + var orderid = newflatteneddigicertinstance["id"].ToString(); + + var digicertmetadataupdateapilocation = "https://www.digicert.com/services/v2/order/certificate/" + orderid + "/custom-field"; + var digicertnewfieldsclient = new RestClient(); + var digicertnewfieldsrequest = new RestRequest(digicertmetadataupdateapilocation); + digicertnewfieldsrequest.AddHeader("Accept", "application/json"); + digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikey); + + foreach (var newfield in fullcustomdgfieldlist) { - payloadforkf.Metadata[metadatanamequery.FirstOrDefault().DigicertFieldName] = metadatafieldinstance.value; + string keyfactorfieldname = ""; + bool datauploaded = false; + //Lookup the keyfactor name for digicert fields + foreach (var sublookup in kfcustomfields) + { + if (sublookup.DigicertFieldName == newfield.label) + { + Dictionary metadatapayload = new Dictionary(); + metadatapayload["metadata_id"] = newfield.id.ToString(); + //Update payload with data + metadatapayload["value"] = kfstoredmetadata[sublookup.DigicertFieldName]; + var newserializedsyncfield = JsonConvert.SerializeObject(metadatapayload); + digicertnewfieldsrequest.AddParameter("application/json", newserializedsyncfield, ParameterType.RequestBody); + var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); + datauploaded = true; + } + } } - + numcertsdatauploaded += 1; } + } + totalcertsprocessed += 1; + } + Console.WriteLine($"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); + Console.WriteLine($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); + logger.LogDebug($"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); + logger.LogDebug($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); + } + + // Syncing Data from DigiCert TO Keyfactor + if (config_mode == "dctokf") + { + Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + logger.LogDebug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + //Each cert that is DigiCert in origin in Keyfactor is looked up on DigiCert via serial number, + //and the metadata contents from those fields are stored. + var digicertlookupclient = new RestClient(); + List digicertcertificates = new List(); + foreach (var certinstance in certlist) + { + var digicertlookupurl = "https://www.digicert.com/services/v2/order/certificate/"; + + var bodytemplate = new RootDigicertLookup(); + var searchcriterioninstance = new SearchCriterion(); + bodytemplate.searchCriteriaList.Add(searchcriterioninstance); + + digicertlookupurl = digicertlookupurl + certinstance.SerialNumber; + var lookuprequest = new RestRequest(digicertlookupurl); + lookuprequest.AddHeader("Content-Type", "application/json"); + lookuprequest.AddHeader("X-DC-DEVKEY", digicertapikey); + var digicertlookupresponse = client.Execute(lookuprequest); + var parseddigicertresponse = JsonConvert.DeserializeObject(digicertlookupresponse.Content); + if (parseddigicertresponse.certificate != null) + { + var flatteneddigicertinstance = ClassConverter(parseddigicertresponse); + digicertcertificates.Add(parseddigicertresponse); } } - var flatteneddigicertinstance = ClassConverter(digicertcertinstance); + Console.WriteLine("Pulled DigiCert matching DigiCert cert data."); + logger.LogDebug("Pulled DigiCert matching DigiCert cert data."); + - //Getting manually selected metadata field values (not custom in Digicert) - foreach (var manualinstance in kfmanualfields) + int certcounttracker = 0; + foreach (var digicertcertinstance in digicertcertificates) { - string[] access = manualinstance.DigicertFieldName.Split("."); + var finalsyncclient = new RestClient(); + finalsyncclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); + var finalsyncurl = keyfactorapilocation + "Certificates/Metadata"; + //Find matching certificate via Keyfactor ID + var query = from kfcertlocal in certlist + where kfcertlocal.SerialNumber == + digicertcertinstance.certificate.serial_number.ToUpper() + select kfcertlocal; + var certificateid = query.FirstOrDefault().Id; - List keys = access.ToList(); - Dictionary recursionresult = recursiveopener(flatteneddigicertinstance, keys, keys.Count); - object value = new object(); - if (recursionresult != null) + + var payloadforkf = new KeyfactorMetadataQuery(); + payloadforkf.Id = certificateid; + + if (digicertcertinstance.custom_fields != null) { - value = recursionresult.First().Value; + // Getting custom metadata field values + foreach (var metadatafieldinstance in digicertcertinstance.custom_fields) + { + if (importallcustomdigicertfields == true) + { + // Using autoimport and thus using autorename + payloadforkf.Metadata[ReplaceAllWhiteSpaces(metadatafieldinstance.label, replacementcharacter)] = metadatafieldinstance.value; + } + else + { + //Using custom names + var metadatanamequery = from customfieldinstance in kfcustomfields + where customfieldinstance.DigicertFieldName == + metadatafieldinstance.label + select customfieldinstance; + if (metadatanamequery.FirstOrDefault() != null) + { + payloadforkf.Metadata[metadatanamequery.FirstOrDefault().DigicertFieldName] = metadatafieldinstance.value; + } + + } + + } } - else + + var flatteneddigicertinstance = ClassConverter(digicertcertinstance); + + //Getting manually selected metadata field values (not custom in DigiCert) + foreach (var manualinstance in kfmanualfields) { - value = ""; + string[] access = manualinstance.DigicertFieldName.Split("."); + + List keys = access.ToList(); + Dictionary recursionresult = recursiveopener(flatteneddigicertinstance, keys, keys.Count); + object value = new object(); + if (recursionresult != null) + { + value = recursionresult.First().Value; + } + else + { + value = ""; + } + payloadforkf.Metadata[manualinstance.KeyfactorMetadataFieldName] = value; } - payloadforkf.Metadata[manualinstance.KeyfactorMetadataFieldName] = value; + payloadforkf.Metadata["DigicertID"] = digicertcertinstance.id.ToString(); + //Sending the payload off to Keyfactor for the update + var finalsyncreq = new RestRequest(finalsyncurl); + finalsyncreq.AddHeader("Content-Type", "application/json"); + finalsyncreq.AddHeader("x-keyfactor-api-version", "1"); + finalsyncreq.AddHeader("x-keyfactor-requested-with", "APIClient"); + var serializedsyncfield = JsonConvert.SerializeObject(payloadforkf); + finalsyncreq.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); + finalsyncclient.Put(finalsyncreq); + ++certcounttracker; } - payloadforkf.Metadata["DigicertID"] = digicertcertinstance.id.ToString(); - //Sending the payload off to Keyfactor for the update - var finalsyncreq = new RestRequest(finalsyncurl); - finalsyncreq.AddHeader("Content-Type", "application/json"); - finalsyncreq.AddHeader("x-keyfactor-api-version", "1"); - finalsyncreq.AddHeader("x-keyfactor-requested-with", "APIClient"); - var serializedsyncfield = JsonConvert.SerializeObject(payloadforkf); - finalsyncreq.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); - finalsyncclient.Put(finalsyncreq); - ++certcounttracker; - } - Console.WriteLine($"Metadata sync complete. Number of certs synced: {certcounttracker.ToString()}"); - logger.LogDebug($"Metadata sync complete. Number of certs synced: {certcounttracker.ToString()}"); + Console.WriteLine($"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); + logger.LogDebug($"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); + } } } } diff --git a/digicert-metadata-sync/Models/DigicertCustomField.cs b/digicert-metadata-sync/Models/DigicertCustomField.cs new file mode 100644 index 0000000..a622ce7 --- /dev/null +++ b/digicert-metadata-sync/Models/DigicertCustomField.cs @@ -0,0 +1,36 @@ +// Copyright 2021 Keyfactor +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace DigicertMetadataSync; + +partial class DigicertSync +{ + + public class DigicertCustomFieldInstance + { + public int id { get; set; } = 999999999; + public string label { get; set; } = ""; + public bool is_required { get; set; } = false; + public bool is_active { get; set; } = true; + public string data_type { get; set; } = "anything"; + public string kf_field_name { get; set; } = ""; + } + + public class DigicertMetadataUpdateInstance + { + public int metadata_id { get; set; } = 999999999; + public string value { get; set; } = "false"; + } + +} \ No newline at end of file diff --git a/digicert-metadata-sync/Models/KeyfactorCertInstance.cs b/digicert-metadata-sync/Models/KeyfactorCertInstance.cs index 680eeb4..11e3670 100644 --- a/digicert-metadata-sync/Models/KeyfactorCertInstance.cs +++ b/digicert-metadata-sync/Models/KeyfactorCertInstance.cs @@ -66,6 +66,7 @@ public class KeyfactorCert public class Metadata { + } public class Detailedkeyusage diff --git a/readme_source.md b/readme_source.md index 2ff84b7..cdb8bcf 100644 --- a/readme_source.md +++ b/readme_source.md @@ -1,16 +1,27 @@ + + ## Overview -This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. +This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. The latest update allows for syncronization of custom field contents from Keyfactor to DigiCert. New fields are created in Keyfactor and DigiCert to accomodate for this. ## Installation and Usage The tool comes as a Windows executable. The tool performs synchronization each time its run. For the tool to run automatically, it needs to be added as a scheduled process using Windows. The advised interval for running it is once per week. The files App.config and manualfields.json need to be present in the same directory as the tool for it to run correctly. The specific location from which the tool is ran does not matter, but it needs to have access to both the Keyfactor API endpoint as well as Digicert, and appropriate permissions for access to the configuration files. An explanation for the settings found in these files is given below. +## Command Line Arguments +One of these two arguments needs to be used for the tool to run. +- "kftodc" +Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. Example: ```.\DigicertMetadataSync.exe kftodc``` +- "dctokf" +Syncronizes the contents of both custom and non-custom fields from DigiCert to Keyfactor. The fields are listed in manualfields.json, and are created if necessary. +Example: ```.\DigicertMetadataSync.exe dctokf``` ## Settings The settings currently present in these files are shown as an example and need to be configured for your specific situation. ### app.config settings - DigicertAPIKey -Standard DigiCert API access key +Standard DigiCert API access key. +- DigicertAPIKeyTopPerm +DigiCert API access key with restrictions set to "None" - required for sync from Keyfactor to DigiCert. - KeyfactorDomainAndUser Same credential as used when logging into Keyfactor Command. A different set of credentials can be used provided they have adequate access permissions. - KeyfactorPassword @@ -53,3 +64,4 @@ String to be input into Keyfactor as the metadata field hint. - KeyfactorAllowAPI Allows API management of this metadata field in Keyfactor. Should be set to true for continuous synchronization with this tool. + From 71b2766a9816aaacfec9dc20ec0223249d702688 Mon Sep 17 00:00:00 2001 From: Keyfactor Date: Fri, 19 May 2023 15:03:12 +0000 Subject: [PATCH 02/12] Update generated README --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 7465f0e..ec98891 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,6 @@ This API client allows for programmatic management of Keyfactor resources. Digicert Metadata Sync is open source and there is **no SLA** for this tool/library/client. Keyfactor will address issues as resources become available. Keyfactor customers may request escalation by opening up a support ticket through their Keyfactor representative. ###### To report a problem or suggest a new feature, use the **[Issues](../../issues)** tab. If you want to contribute actual bug fixes or proposed enhancements, use the **[Pull requests](../../pulls)** tab. -___ From 3c05ef7259fafeee050e2dddbde13af6614ba523 Mon Sep 17 00:00:00 2001 From: Mark Kachkaev <37276742+mkachk@users.noreply.github.com> Date: Wed, 7 Jun 2023 10:20:25 -0400 Subject: [PATCH 03/12] Wb release 2.0.2 (#14) * Fixed issue with no input for either field type leading to a crash. Two other bugs fixed. * Update CHANGELOG.md --- CHANGELOG.md | 5 ++ .../AddFieldsToKeyfactor.cs | 79 ++++++++++--------- digicert-metadata-sync/Helpers.cs | 35 ++++---- digicert-metadata-sync/MetadataSync.cs | 39 +++++++-- .../Models/InternalClasses.cs | 12 +-- 5 files changed, 102 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8247963..87cd12a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +Version 2.0.2 +- Fixed issue with no input for either custom or manual fields leading to a crash. +- Fixed issue with data for imported DigiCert fields renamed with a replacement character not syncing back to DigiCert. +- Fixed possible crash caused by importing DigiCert custom fields with "Anything" data type. + Version 2.0 - Added ability to sync custom fields from Keyfactor to DigiCert. diff --git a/digicert-metadata-sync/AddFieldsToKeyfactor.cs b/digicert-metadata-sync/AddFieldsToKeyfactor.cs index 363da96..44af1b6 100644 --- a/digicert-metadata-sync/AddFieldsToKeyfactor.cs +++ b/digicert-metadata-sync/AddFieldsToKeyfactor.cs @@ -31,15 +31,49 @@ public static Tuple> AddFieldsToKeyfactor(List newfields = new List(); - foreach (var metadatainstance in inputlist) + if (inputlist.Count != 0) { - if (noexistingfields == false) + foreach (var metadatainstance in inputlist) { - var fieldquery = from existingmetadatainstance in existingmetadatalist - where existingmetadatainstance.Name == metadatainstance.Name - select existingmetadatainstance; - // If field does not exist in Keyfactor, add it. - if (!fieldquery.Any()) + if (noexistingfields == false) + { + var fieldquery = from existingmetadatainstance in existingmetadatalist + where existingmetadatainstance.Name == metadatainstance.Name + select existingmetadatainstance; + // If field does not exist in Keyfactor, add it. + if (!fieldquery.Any()) + { + var addfieldrequest = new RestRequest(addfieldstokeyfactorurl); + addfieldrequest.AddHeader("Content-Type", "application/json"); + addfieldrequest.AddHeader("Accept", "application/json"); + addfieldrequest.AddHeader("x-keyfactor-api-version", "1"); + addfieldrequest.AddHeader("x-keyfactor-requested-with", "APIClient"); + var serializedfield = JsonConvert.SerializeObject(metadatainstance); + addfieldrequest.AddParameter("application/json", serializedfield, ParameterType.RequestBody); + RestResponse metadataresponse = new RestResponse(); + try + { + metadataresponse = addfieldsclient.Post(addfieldrequest); + newfields.Add(metadatainstance.Name); + ++totalnumberadded; + } + catch (HttpRequestException e) + { + Console.WriteLine(metadataresponse.Content); + throw e; + } + } + else + { + if (fieldquery.FirstOrDefault().DataType != metadatainstance.DataType) + { + //Throw error if datatype included in keyfactor does not match the digicert one. + NotSupportedException mismatchedtypes = new NotSupportedException(); + throw mismatchedtypes; + } + } + } + else { var addfieldrequest = new RestRequest(addfieldstokeyfactorurl); addfieldrequest.AddHeader("Content-Type", "application/json"); @@ -52,7 +86,6 @@ public static Tuple> AddFieldsToKeyfactor(List> AddFieldsToKeyfactor(List> returnvals = new Tuple>(totalnumberadded, newfields); diff --git a/digicert-metadata-sync/Helpers.cs b/digicert-metadata-sync/Helpers.cs index b44c192..77f7ff0 100644 --- a/digicert-metadata-sync/Helpers.cs +++ b/digicert-metadata-sync/Helpers.cs @@ -82,25 +82,28 @@ private static List convertlisttokf(List formattedlist = new List(); - foreach (ReadInMetadataField input in inputlist) + if (inputlist.Count != 0) { - KeyfactorMetadataInstanceSendoff formatinstance = new KeyfactorMetadataInstanceSendoff(); - if (input.KeyfactorMetadataFieldName == null || input.KeyfactorMetadataFieldName == "") + foreach (ReadInMetadataField input in inputlist) { - //If name is emtpy, use autocomplete. - formatinstance.Name = ReplaceAllWhiteSpaces(input.DigicertFieldName, replacementcharacter); - } - else - { - //Use user input preferred name. - formatinstance.Name = input.KeyfactorMetadataFieldName; - } + KeyfactorMetadataInstanceSendoff formatinstance = new KeyfactorMetadataInstanceSendoff(); + if (input.KeyfactorMetadataFieldName == null || input.KeyfactorMetadataFieldName == "") + { + //If name is emtpy, use autocomplete. + formatinstance.Name = ReplaceAllWhiteSpaces(input.DigicertFieldName, replacementcharacter); + } + else + { + //Use user input preferred name. + formatinstance.Name = input.KeyfactorMetadataFieldName; + } - formatinstance.AllowAPI = Convert.ToBoolean(input.KeyfactorAllowAPI); - formatinstance.Hint = input.KeyfactorHint; - formatinstance.DataType = TypeMatcher(input.KeyfactorDataType); - formatinstance.Description = input.KeyfactorDescription; - formattedlist.Add(formatinstance); + formatinstance.AllowAPI = Convert.ToBoolean(input.KeyfactorAllowAPI); + formatinstance.Hint = input.KeyfactorHint; + formatinstance.DataType = TypeMatcher(input.KeyfactorDataType); + formatinstance.Description = input.KeyfactorDescription; + formattedlist.Add(formatinstance); + } } return formattedlist; diff --git a/digicert-metadata-sync/MetadataSync.cs b/digicert-metadata-sync/MetadataSync.cs index 1e36077..389426b 100644 --- a/digicert-metadata-sync/MetadataSync.cs +++ b/digicert-metadata-sync/MetadataSync.cs @@ -103,11 +103,11 @@ public static void Main(string[] args) var kfdatatype = "String"; if (customdigicertmetadatafieldlist[i].data_type != null) { - kfdatatype = customdigicertmetadatafieldlist[i].data_type; + localkffieldinstance.KeyfactorDataType = customdigicertmetadatafieldlist[i].data_type; } else { - kfdatatype = "String"; + localkffieldinstance.KeyfactorDataType = "String"; } if (customdigicertmetadatafieldlist[i].label != null) { @@ -119,10 +119,22 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. localkffieldinstance.DigicertFieldName = customdigicertmetadatafieldlist[i].label; localkffieldinstance.KeyfactorMetadataFieldName = ReplaceAllWhiteSpaces(customdigicertmetadatafieldlist[i].label, replacementcharacter); } + else + { + localkffieldinstance.DigicertFieldName = ""; + localkffieldinstance.KeyfactorMetadataFieldName = ""; + } if (customdigicertmetadatafieldlist[i].description != null) { localkffieldinstance.KeyfactorDescription = customdigicertmetadatafieldlist[i].description; } + else + { + localkffieldinstance.KeyfactorDescription = "None."; + } + + localkffieldinstance.KeyfactorAllowAPI = "True"; + localkffieldinstance.KeyfactorHint = ""; //Other parameters like enrollment can be set here too. kfcustomfields.Add(localkffieldinstance); @@ -134,12 +146,20 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. // Converts blank fields etc and preps the data. var customfieldslst = "CustomFields"; kfcustomfields = config.GetSection(customfieldslst).Get>(); + if (kfcustomfields == null) + { + kfcustomfields = new List(); + } } //Adding metadata fields for the ID and the email of the requester from DigiCert. List kfmanualfields = new List(); var manualfieldslist = "ManualFields"; kfmanualfields = config.GetSection(manualfieldslist).Get>(); + if (kfmanualfields == null) + { + kfmanualfields = new List(); + } logger.LogDebug("Performed field conversion."); //Pulling list of existing metadata fields from Keyfactor for later comparison. @@ -323,12 +343,15 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. { Dictionary metadatapayload = new Dictionary(); metadatapayload["metadata_id"] = newfield.id.ToString(); - //Update payload with data - metadatapayload["value"] = kfstoredmetadata[sublookup.DigicertFieldName]; - var newserializedsyncfield = JsonConvert.SerializeObject(metadatapayload); - digicertnewfieldsrequest.AddParameter("application/json", newserializedsyncfield, ParameterType.RequestBody); - var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); - datauploaded = true; + if (kfstoredmetadata.ContainsKey(sublookup.KeyfactorMetadataFieldName)) + { + metadatapayload["value"] = kfstoredmetadata[sublookup.KeyfactorMetadataFieldName]; + var newserializedsyncfield = JsonConvert.SerializeObject(metadatapayload); + digicertnewfieldsrequest.AddParameter("application/json", newserializedsyncfield, ParameterType.RequestBody); + var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); + datauploaded = true; + } + } } } diff --git a/digicert-metadata-sync/Models/InternalClasses.cs b/digicert-metadata-sync/Models/InternalClasses.cs index c378a2e..0bdd25a 100644 --- a/digicert-metadata-sync/Models/InternalClasses.cs +++ b/digicert-metadata-sync/Models/InternalClasses.cs @@ -28,12 +28,12 @@ public class CustomDigicertMetadataInstance public class ReadInMetadataField { - public string DigicertFieldName { get; set; } - public string KeyfactorMetadataFieldName { get; set; } - public string KeyfactorDescription { get; set; } - public string KeyfactorDataType { get; set; } - public string KeyfactorHint { get; set; } - public string KeyfactorAllowAPI { get; set; } + public string DigicertFieldName { get; set; } = "local_test_nullx0"; + public string KeyfactorMetadataFieldName { get; set; } = "test_name_nullx0"; + public string KeyfactorDescription { get; set; } = "None."; + public string KeyfactorDataType { get; set; } = "string"; + public string KeyfactorHint { get; set; } = "None."; + public string KeyfactorAllowAPI { get; set; } = "True"; } public class KeyfactorMetadataInstanceSendoff From 256a8e0fe39870afcce1e52685899febd4a449d0 Mon Sep 17 00:00:00 2001 From: Mikey Henderson Date: Wed, 7 Jun 2023 07:25:41 -0700 Subject: [PATCH 04/12] [skip ci] Update CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87cd12a..472b4c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,9 @@ -Version 2.0.2 +Version 2.0.1 - Fixed issue with no input for either custom or manual fields leading to a crash. - Fixed issue with data for imported DigiCert fields renamed with a replacement character not syncing back to DigiCert. - Fixed possible crash caused by importing DigiCert custom fields with "Anything" data type. -Version 2.0 +Version 2.0.0 - Added ability to sync custom fields from Keyfactor to DigiCert. - Tool now requires command line argument to specify sync direction: "dctokf" for DigiCert to Keyfactor and "kftodc" for Keyfactor to DigiCert. From fdfb502cc120c5b76f0af7cca2633c4fa959a338 Mon Sep 17 00:00:00 2001 From: Mark Kachkaev <37276742+mkachk@users.noreply.github.com> Date: Thu, 20 Jul 2023 11:52:46 -0400 Subject: [PATCH 05/12] Fixes ab#47259 * Fixed issue with additional_emails field not syncing. * Added independent logging via NLog. --- CHANGELOG.md | 20 +- README.md | 23 +- .../AddFieldsToKeyfactor.cs | 21 +- digicert-metadata-sync/App.config | 4 +- .../DigicertMetadataSync.csproj | 14 +- .../GrabCustomFieldsFromDigiCert.cs | 3 +- digicert-metadata-sync/Helpers.cs | 88 +- digicert-metadata-sync/MetadataSync.cs | 796 +++++++++--------- .../Models/DigicertCertInstance.cs | 3 +- .../Models/DigicertCustomField.cs | 6 +- .../Models/DigicertLookup.cs | 6 +- .../Models/InternalClasses.cs | 2 +- .../Models/KeyfactorCertInstance.cs | 3 +- .../Models/KeyfactorMetadataInstance.cs | 2 +- .../Models/KeyfactorMetadataQuery.cs | 4 +- digicert-metadata-sync/NLog.config | 21 + readme_source.md | 21 +- 17 files changed, 481 insertions(+), 556 deletions(-) create mode 100644 digicert-metadata-sync/NLog.config diff --git a/CHANGELOG.md b/CHANGELOG.md index 472b4c8..5552b98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,20 @@ +Version 2.0.2 + + Fixed issue with additional_emails field not syncing. + Added independent logging via NLog. + Version 2.0.1 -- Fixed issue with no input for either custom or manual fields leading to a crash. -- Fixed issue with data for imported DigiCert fields renamed with a replacement character not syncing back to DigiCert. -- Fixed possible crash caused by importing DigiCert custom fields with "Anything" data type. + + Fixed issue with no input for either custom or manual fields leading to a crash. + Fixed issue with data for imported DigiCert fields renamed with a replacement character not syncing back to DigiCert. + Fixed possible crash caused by importing DigiCert custom fields with "Anything" data type. Version 2.0.0 -- Added ability to sync custom fields from Keyfactor to DigiCert. -- Tool now requires command line argument to specify sync direction: "dctokf" for DigiCert to Keyfactor and "kftodc" for Keyfactor to DigiCert. -- New DigiCert API Key with restrictions set to "None" in DigiCert config required to perform sync from Keyfactor to Digicert. + Added ability to sync custom fields from Keyfactor to DigiCert. + Tool now requires command line argument to specify sync direction: "dctokf" for DigiCert to Keyfactor and "kftodc" for Keyfactor to DigiCert. + New DigiCert API Key with restrictions set to "None" in DigiCert config required to perform sync from Keyfactor to Digicert. Version 1.0 -- Initial Release + Initial Release diff --git a/README.md b/README.md index ec98891..4ca324c 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,7 @@ A tool to automatically synchronize metadata fields and their content from DigiC #### Integration status: Production - Ready for use in production environments. -## About the Keyfactor API Client -This API client allows for programmatic management of Keyfactor resources. @@ -19,30 +17,19 @@ Digicert Metadata Sync is open source and there is **no SLA** for this tool/libr - - ## Overview -This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. The latest update allows for syncronization of custom field contents from Keyfactor to DigiCert. New fields are created in Keyfactor and DigiCert to accomodate for this. +This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. ## Installation and Usage -The tool comes as a Windows executable. The tool performs synchronization each time its run. For the tool to run automatically, it needs to be added as a scheduled process using Windows. The advised interval for running it is once per week. The files App.config and manualfields.json need to be present in the same directory as the tool for it to run correctly. The specific location from which the tool is ran does not matter, but it needs to have access to both the Keyfactor API endpoint as well as Digicert, and appropriate permissions for access to the configuration files. +The tool comes as a Windows executable. The tool performs synchronization each time its run. For the tool to run automatically, it needs to be added as a scheduled process using Windows. The advised interval for running it is once per week. The files DigicertMetadataSync.dll.config and manualfields.json need to be present in the same directory as the tool for it to run correctly. The specific location from which the tool is ran does not matter, but it needs to have access to both the Keyfactor API endpoint as well as Digicert, and appropriate permissions for access to the configuration files. An explanation for the settings found in these files is given below. -## Command Line Arguments -One of these two arguments needs to be used for the tool to run. -- "kftodc" -Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. Example: ```.\DigicertMetadataSync.exe kftodc``` -- "dctokf" -Syncronizes the contents of both custom and non-custom fields from DigiCert to Keyfactor. The fields are listed in manualfields.json, and are created if necessary. -Example: ```.\DigicertMetadataSync.exe dctokf``` ## Settings The settings currently present in these files are shown as an example and need to be configured for your specific situation. -### app.config settings +### DigicertMetadataSync.dll.config settings - DigicertAPIKey -Standard DigiCert API access key. -- DigicertAPIKeyTopPerm -DigiCert API access key with restrictions set to "None" - required for sync from Keyfactor to DigiCert. +Standard DigiCert API access key - KeyfactorDomainAndUser Same credential as used when logging into Keyfactor Command. A different set of credentials can be used provided they have adequate access permissions. - KeyfactorPassword @@ -86,4 +73,6 @@ String to be input into Keyfactor as the metadata field hint. - KeyfactorAllowAPI Allows API management of this metadata field in Keyfactor. Should be set to true for continuous synchronization with this tool. +### Logging +Logging functionality can be configured via entering either "Debug" or "Trace" into the value of `` in NLog.config. diff --git a/digicert-metadata-sync/AddFieldsToKeyfactor.cs b/digicert-metadata-sync/AddFieldsToKeyfactor.cs index 44af1b6..8425582 100644 --- a/digicert-metadata-sync/AddFieldsToKeyfactor.cs +++ b/digicert-metadata-sync/AddFieldsToKeyfactor.cs @@ -20,21 +20,19 @@ namespace DigicertMetadataSync; // This fuction adds the fields to keyfactor. // It will only add new fields. -partial class DigicertSync +internal partial class DigicertSync { - public static Tuple> AddFieldsToKeyfactor(List inputlist, + public static Tuple> AddFieldsToKeyfactor(List inputlist, List existingmetadatalist, bool noexistingfields, string keyfactorusername, string keyfactorpassword, string keyfactorapilocation) { var addfieldstokeyfactorurl = keyfactorapilocation + "MetadataFields"; var addfieldsclient = new RestClient(); addfieldsclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); - int totalnumberadded = 0; - List newfields = new List(); + var totalnumberadded = 0; + var newfields = new List(); if (inputlist.Count != 0) - { foreach (var metadatainstance in inputlist) - { if (noexistingfields == false) { var fieldquery = from existingmetadatainstance in existingmetadatalist @@ -50,7 +48,7 @@ public static Tuple> AddFieldsToKeyfactor(List> AddFieldsToKeyfactor(List> AddFieldsToKeyfactor(List> AddFieldsToKeyfactor(List> returnvals = new Tuple>(totalnumberadded, newfields); + + var returnvals = new Tuple>(totalnumberadded, newfields); return returnvals; } diff --git a/digicert-metadata-sync/App.config b/digicert-metadata-sync/App.config index 3d97fda..301d1d0 100644 --- a/digicert-metadata-sync/App.config +++ b/digicert-metadata-sync/App.config @@ -5,10 +5,10 @@ - + - + \ No newline at end of file diff --git a/digicert-metadata-sync/DigicertMetadataSync.csproj b/digicert-metadata-sync/DigicertMetadataSync.csproj index 2195e9c..11cf61a 100644 --- a/digicert-metadata-sync/DigicertMetadataSync.csproj +++ b/digicert-metadata-sync/DigicertMetadataSync.csproj @@ -10,11 +10,12 @@ - - - - - + + + + + + PreserveNewest @@ -24,6 +25,9 @@ PreserveNewest + + Always + diff --git a/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs b/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs index 834097e..0236336 100644 --- a/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs +++ b/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs @@ -26,7 +26,6 @@ partial class DigicertSync { public static List GrabCustomFieldsFromDigiCert(string apikey) { - ILogger logger = LogHandler.GetClassLogger(); var digicertclient = new RestClient(); var customfieldsretrieval = "https://www.digicert.com/services/v2/account/metadata"; var digicertrequest = new RestRequest(customfieldsretrieval); @@ -38,7 +37,7 @@ public static List GrabCustomFieldsFromDigiCert( trimmeddigicertresponse = trimmeddigicertresponse.Remove(lengthofresponse - 1, 1); var fieldlist = JsonConvert.DeserializeObject>(trimmeddigicertresponse); Console.WriteLine("Obtained custom fields from DigiCert."); - logger.LogDebug("Obtained custom fields from DigiCert."); + _logger.Debug("Obtained custom fields from DigiCert."); return fieldlist; } } \ No newline at end of file diff --git a/digicert-metadata-sync/Helpers.cs b/digicert-metadata-sync/Helpers.cs index 77f7ff0..501e155 100644 --- a/digicert-metadata-sync/Helpers.cs +++ b/digicert-metadata-sync/Helpers.cs @@ -12,57 +12,46 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System.Reflection; using System.Text.RegularExpressions; +using Newtonsoft.Json.Linq; namespace DigicertMetadataSync; -partial class DigicertSync +internal partial class DigicertSync { public static int TypeMatcher(string digicerttype) { if (digicerttype.Contains("int") || digicerttype.Contains("Int")) - { // 2 matches the keyfactor int type metadata field return 2; - } - else - { - //1 matches the keyfactor string type - return 1; - } + //1 matches the keyfactor string type + return 1; } } -partial class DigicertSync +internal partial class DigicertSync { public static Dictionary ClassConverter(object obj) { if (obj != null && obj != "") { - Dictionary resultdict = new Dictionary(); + var resultdict = new Dictionary(); var propertylist = obj.GetType().GetProperties(); - foreach (PropertyInfo prop in propertylist) + foreach (var prop in propertylist) { - string propName = prop.Name; + var propName = prop.Name; var val = obj.GetType().GetProperty(propName).GetValue(obj, null); if (val != null) - { resultdict.Add(propName, val); - } else - { resultdict.Add(propName, ""); - } } return resultdict; } - else - { - return null; - } + + return null; } public static string ReplaceAllWhiteSpaces(string str, string replacement) @@ -72,31 +61,24 @@ public static string ReplaceAllWhiteSpaces(string str, string replacement) public static bool CheckMode(string mode) { - if ((mode == "kftodc") || (mode == "dctokf")){ - return true; - } + if (mode == "kftodc" || mode == "dctokf") return true; return false; } private static List convertlisttokf(List inputlist, string replacementcharacter) { - List formattedlist = new List(); + var formattedlist = new List(); if (inputlist.Count != 0) - { - foreach (ReadInMetadataField input in inputlist) + foreach (var input in inputlist) { - KeyfactorMetadataInstanceSendoff formatinstance = new KeyfactorMetadataInstanceSendoff(); + var formatinstance = new KeyfactorMetadataInstanceSendoff(); if (input.KeyfactorMetadataFieldName == null || input.KeyfactorMetadataFieldName == "") - { //If name is emtpy, use autocomplete. formatinstance.Name = ReplaceAllWhiteSpaces(input.DigicertFieldName, replacementcharacter); - } else - { //Use user input preferred name. formatinstance.Name = input.KeyfactorMetadataFieldName; - } formatinstance.AllowAPI = Convert.ToBoolean(input.KeyfactorAllowAPI); formatinstance.Hint = input.KeyfactorHint; @@ -104,46 +86,22 @@ private static List convertlisttokf(List recursiveopener(Dictionary dictin, List keys, - int limit) + public static JObject Flatten(JObject jObject, string parentName = "") { - // Recursion steps - // Each iteration: access dict at keys[0], pop back off keys. K - object result = new object(); - if (dictin == null) + var result = new JObject(); + foreach (var property in jObject.Properties()) { - return null; - } - - if (limit != 0) - { - var location = keys[0]; - keys.RemoveAt(0); - --limit; - if (limit == 0) - { - //At the bottom of keys - Dictionary newdict = new Dictionary(); - newdict[location] = dictin[location]; - return newdict; - } + var propName = string.IsNullOrEmpty(parentName) ? property.Name : $"{parentName}.{property.Name}"; + if (property.Value is JObject nestedObject) + result.Merge(Flatten(nestedObject, propName)); else - { - //Keep recursion - - Dictionary newdict = ClassConverter(dictin[location]); - return recursiveopener(newdict, keys, limit); - } - } - else - { - //Found nothing. - return null; + result[propName] = property.Value; } + + return result; } } \ No newline at end of file diff --git a/digicert-metadata-sync/MetadataSync.cs b/digicert-metadata-sync/MetadataSync.cs index 389426b..380c05b 100644 --- a/digicert-metadata-sync/MetadataSync.cs +++ b/digicert-metadata-sync/MetadataSync.cs @@ -5,483 +5,445 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions // and limitations under the License. - -using System; -using System.Threading; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json; +using NLog; using RestSharp; using RestSharp.Authenticators; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using System.Text.RegularExpressions; -using System; -using System.Configuration; -using System.Reflection; -using Microsoft.Extensions.Configuration; -using System.Collections.Specialized; -using System.Text.Json.Serialization.Metadata; -using Keyfactor; -using Microsoft.Extensions.Logging; -using Keyfactor.Logging; -using static DigicertMetadataSync.DigicertSync; -using System.Collections.Immutable; -using System.Runtime.CompilerServices; - -namespace DigicertMetadataSync +using ConfigurationManager = System.Configuration.ConfigurationManager; + +namespace DigicertMetadataSync; + +internal partial class DigicertSync { - partial class DigicertSync + // create a static _logger field + private static readonly Logger _logger = LogManager.GetCurrentClassLogger(); + + public static void Main(string[] args) { + _logger.Debug("Start sync"); + var digicertapikey = ConfigurationManager.AppSettings.Get("DigicertAPIKey"); + var digicertapikeytopperm = ConfigurationManager.AppSettings.Get("DigicertAPIKeyTopPerm"); + var keyfactorusername = ConfigurationManager.AppSettings.Get("KeyfactorDomainAndUser"); + var keyfactorpassword = ConfigurationManager.AppSettings.Get("KeyfactorPassword"); + var replacementcharacter = ConfigurationManager.AppSettings.Get("ReplaceDigicertWhiteSpaceCharacterInName"); + var importallcustomdigicertfields = + Convert.ToBoolean(ConfigurationManager.AppSettings.Get("ImportAllCustomDigicertFields")); + _logger.Debug("Settings: importallcustomdigicertfields={0}, replacementcharacter={1}", + importallcustomdigicertfields, replacementcharacter); + var config_mode = args[0]; + if (CheckMode(config_mode) == false) + { + _logger.Error("Inappropriate configuration mode. Check your command line arguments."); + throw new Exception("Inappropriate configuration mode. Check your command line arguments."); + } - public static void Main(string[] args) + //Get list of all DigiCert certs from Keyfactor based on query that contains DigiCert as issuer. + var returnlimit = ConfigurationManager.AppSettings.Get("KeyfactorCertSearchReturnLimit"); + var keyfactorapilocation = ConfigurationManager.AppSettings.Get("KeyfactorAPIEndpoint"); + var digicertIssuerQueryterm = ConfigurationManager.AppSettings.Get("KeyfactorDigicertIssuedCertQueryTerm"); + _logger.Debug($"Loaded config. Processing with a Keyfactor Query Return Limit of {returnlimit}"); + + var digicertlookup = keyfactorapilocation + "Certificates?pq.queryString=IssuerDN%20-contains%20%22" + + digicertIssuerQueryterm + "%22&pq.returnLimit=" + returnlimit + + "&includeMetadata=true"; + var client = new RestClient(); + client.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); + var request = new RestRequest(digicertlookup); + request.AddHeader("Accept", "application/json"); + request.AddHeader("x-keyfactor-api-version", "1"); + request.AddHeader("x-keyfactor-requested-with", "APIClient"); + var response = client.Execute(request); + var rawresponse = response.Content; + var certlist = JsonConvert.DeserializeObject>(rawresponse, + new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + Console.WriteLine("Got DigiCert issued certs from keyfactor"); + _logger.Debug("Got DigiCert issued certs from keyfactor"); + + //Getting list of custom metadata fields from Keyfactor + var getmetadalistkf = keyfactorapilocation + "MetadataFields"; + var getmetadatakfclient = new RestClient(); + getmetadatakfclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); + var metadatakfrequest = new RestRequest(getmetadalistkf); + metadatakfrequest.AddHeader("Accept", "application/json"); + metadatakfrequest.AddHeader("x-keyfactor-api-version", "1"); + metadatakfrequest.AddHeader("x-keyfactor-requested-with", "APIClient"); + var metadatakfresponse = client.Execute(metadatakfrequest); + var metadatakfrawresponse = metadatakfresponse.Content; + var kfmetadatafields = JsonConvert.DeserializeObject>(metadatakfrawresponse); + Console.WriteLine("Got list of custom fields from Keyfactor."); + _logger.Debug("Got list of custom fields from Keyfactor."); + + //Getting list of custom metadata fields on DigiCert + var customdigicertmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey); + + //Convert DigiCert custom fields to Keyfactor appropriate ones + //This depends on whether the setting to import all fields was enabled or not + + var config = new ConfigurationBuilder().SetBasePath(AppDomain.CurrentDomain.BaseDirectory) + .AddJsonFile("manualfields.json").Build(); + var kfcustomfields = new List(); + if (importallcustomdigicertfields) { - ILogger logger = LogHandler.GetClassLogger(); - logger.LogDebug("Start sync"); - var digicertapikey = System.Configuration.ConfigurationManager.AppSettings.Get("DigicertAPIKey"); - var digicertapikeytopperm = System.Configuration.ConfigurationManager.AppSettings.Get("DigicertAPIKeyTopPerm"); - var keyfactorusername = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorDomainAndUser"); - var keyfactorpassword = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorPassword"); - var replacementcharacter = System.Configuration.ConfigurationManager.AppSettings.Get("ReplaceDigicertWhiteSpaceCharacterInName"); - var importallcustomdigicertfields = Convert.ToBoolean(System.Configuration.ConfigurationManager.AppSettings.Get("ImportAllCustomDigicertFields")); - - var config_mode = args[0]; - if (CheckMode(config_mode) == false) + _logger.Debug("Loading custom fields using autofill"); + //This imports all the custom fields based on the list of metadata from DigiCert and does autofill + for (var i = 0; i < customdigicertmetadatafieldlist.Count; i++) { - logger.LogDebug("Inappropriate configuration mode. Check your command line arguments."); - throw new Exception("Inappropriate configuration mode. Check your command line arguments."); + var localkffieldinstance = new ReadInMetadataField(); + var kfdatatype = "String"; + if (customdigicertmetadatafieldlist[i].data_type != null) + localkffieldinstance.KeyfactorDataType = customdigicertmetadatafieldlist[i].data_type; + else + localkffieldinstance.KeyfactorDataType = "String"; + if (customdigicertmetadatafieldlist[i].label != null) + { + /* + NOTICE: KEYFACTOR DOES NOT SUPPORT SPACES IN METADATA FIELD NAMES. + WHITESPACE MUST BE REMOVED FROM THE NAME. + CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. + */ + localkffieldinstance.DigicertFieldName = customdigicertmetadatafieldlist[i].label; + localkffieldinstance.KeyfactorMetadataFieldName = + ReplaceAllWhiteSpaces(customdigicertmetadatafieldlist[i].label, replacementcharacter); + _logger.Debug("DC field name {0} becomes {1} in Keyfactor", localkffieldinstance.DigicertFieldName, + localkffieldinstance.KeyfactorMetadataFieldName); + } + else + { + localkffieldinstance.DigicertFieldName = ""; + localkffieldinstance.KeyfactorMetadataFieldName = ""; + } + + if (customdigicertmetadatafieldlist[i].description != null) + localkffieldinstance.KeyfactorDescription = customdigicertmetadatafieldlist[i].description; + else + localkffieldinstance.KeyfactorDescription = "None."; + + localkffieldinstance.KeyfactorAllowAPI = "True"; + localkffieldinstance.KeyfactorHint = ""; + //Other parameters like enrollment can be set here too. + + kfcustomfields.Add(localkffieldinstance); } + } + else + { + // This loads custom metadata using the manualfields config. + // Converts blank fields etc and preps the data. + var customfieldslst = "CustomFields"; + kfcustomfields = config.GetSection(customfieldslst).Get>(); + if (kfcustomfields == null) kfcustomfields = new List(); + _logger.Debug("Loading custom fields using json, no autofill/conversion"); + } - //Get list of all DigiCert certs from Keyfactor based on query that contains DigiCert as issuer. - var returnlimit = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorCertSearchReturnLimit").ToString(); - var keyfactorapilocation = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorAPIEndpoint").ToString(); - var digicertIssuerQueryterm = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorDigicertIssuedCertQueryTerm").ToString(); - logger.LogDebug($"Loaded config. Processing with a Keyfactor Query Return Limit of {returnlimit.ToString()}"); - - var digicertlookup = keyfactorapilocation + "Certificates?pq.queryString=IssuerDN%20-contains%20%22" - + digicertIssuerQueryterm + "%22&pq.returnLimit=" + returnlimit + "&includeMetadata=true"; - var client = new RestClient(); - client.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); - var request = new RestRequest(digicertlookup); - request.AddHeader("Accept", "application/json"); - request.AddHeader("x-keyfactor-api-version", "1"); - request.AddHeader("x-keyfactor-requested-with", "APIClient"); - var response = client.Execute(request); - var rawresponse = response.Content; - var certlist = JsonConvert.DeserializeObject>(rawresponse, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - Console.WriteLine("Got DigiCert issued certs from keyfactor"); - logger.LogDebug("Got DigiCert issued certs from keyfactor"); - - //Getting list of custom metadata fields from Keyfactor - var getmetadalistkf = keyfactorapilocation + "MetadataFields"; - var getmetadatakfclient = new RestClient(); - getmetadatakfclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); - var metadatakfrequest = new RestRequest(getmetadalistkf); - metadatakfrequest.AddHeader("Accept", "application/json"); - metadatakfrequest.AddHeader("x-keyfactor-api-version", "1"); - metadatakfrequest.AddHeader("x-keyfactor-requested-with", "APIClient"); - var metadatakfresponse = client.Execute(metadatakfrequest); - var metadatakfrawresponse = metadatakfresponse.Content; - var kfmetadatafields = JsonConvert.DeserializeObject>(metadatakfrawresponse); - Console.WriteLine("Got list of custom fields from Keyfactor."); - logger.LogDebug("Got list of custom fields from Keyfactor."); + //Adding metadata fields for the ID and the email of the requester from DigiCert. + var kfmanualfields = new List(); + var manualfieldslist = "ManualFields"; + kfmanualfields = config.GetSection(manualfieldslist).Get>(); + if (kfmanualfields == null) kfmanualfields = new List(); + _logger.Debug("Performed field conversion."); + + //Pulling list of existing metadata fields from Keyfactor for later comparison. + var noexistingfields = true; + + var existingmetadataurl = keyfactorapilocation + "MetadataFields"; + var existingmetadataclient = new RestClient(); + existingmetadataclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); + var existingmetadatareq = new RestRequest(existingmetadataurl); + existingmetadatareq.AddHeader("Accept", "application/json"); + existingmetadatareq.AddHeader("x-keyfactor-api-version", "1"); + existingmetadatareq.AddHeader("x-keyfactor-requested-with", "APIClient"); + var existingmetadataresponse = existingmetadataclient.Execute(existingmetadatareq); + var existingmetadatalist = new List(); + if (existingmetadataresponse != null) + { + //Fields exist + existingmetadatalist = + JsonConvert.DeserializeObject>(existingmetadataresponse.Content); + noexistingfields = false; + } - //Getting list of custom metadata fields on DigiCert - var customdigicertmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey); + Console.WriteLine("Pulled existing metadata fields from keyfactor."); + _logger.Debug("Pulled existing metadata fields from Keyfactor."); + // Converting the read in fields into sendable lists + var convertedmanualfields = convertlisttokf(kfmanualfields, replacementcharacter); + var convertedcustomfields = convertlisttokf(kfcustomfields, replacementcharacter); - //Convert DigiCert custom fields to Keyfactor appropriate ones - //This depends on whether the setting to import all fields was enabled or not + _logger.Trace("Sending following manual fields to KF: {0}", JsonConvert.SerializeObject(convertedmanualfields)); + var totalfieldsadded = 0; - var config = new ConfigurationBuilder().SetBasePath(AppDomain.CurrentDomain.BaseDirectory).AddJsonFile("manualfields.json").Build(); - var kfcustomfields = new List(); - if (importallcustomdigicertfields == true) - { - //This imports all the custom fields based on the list of metadata from DigiCert and does autofill - for (int i = 0; i < customdigicertmetadatafieldlist.Count; i++) - { - var localkffieldinstance = new ReadInMetadataField(); - var kfdatatype = "String"; - if (customdigicertmetadatafieldlist[i].data_type != null) - { - localkffieldinstance.KeyfactorDataType = customdigicertmetadatafieldlist[i].data_type; - } - else - { - localkffieldinstance.KeyfactorDataType = "String"; - } - if (customdigicertmetadatafieldlist[i].label != null) - { - /* - NOTICE: KEYFACTOR DOES NOT SUPPORT SPACES IN METADATA FIELD NAMES. - WHITESPACE MUST BE REMOVED FROM THE NAME. - CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. - */ - localkffieldinstance.DigicertFieldName = customdigicertmetadatafieldlist[i].label; - localkffieldinstance.KeyfactorMetadataFieldName = ReplaceAllWhiteSpaces(customdigicertmetadatafieldlist[i].label, replacementcharacter); - } - else - { - localkffieldinstance.DigicertFieldName = ""; - localkffieldinstance.KeyfactorMetadataFieldName = ""; - } - if (customdigicertmetadatafieldlist[i].description != null) - { - localkffieldinstance.KeyfactorDescription = customdigicertmetadatafieldlist[i].description; - } - else - { - localkffieldinstance.KeyfactorDescription = "None."; - } + //If all the fields are absent from Keyfactor, the fields are added. + var manualresult = AddFieldsToKeyfactor(convertedmanualfields, existingmetadatalist, noexistingfields, + keyfactorusername, keyfactorpassword, keyfactorapilocation); + _logger.Trace("Sending following custom fields to KF: {0}", JsonConvert.SerializeObject(convertedcustomfields)); - localkffieldinstance.KeyfactorAllowAPI = "True"; - localkffieldinstance.KeyfactorHint = ""; - //Other parameters like enrollment can be set here too. + var customresult = AddFieldsToKeyfactor(convertedcustomfields, existingmetadatalist, noexistingfields, + keyfactorusername, keyfactorpassword, keyfactorapilocation); - kfcustomfields.Add(localkffieldinstance); - } - } - else - { - // This loads custom metadata using the manualfields config. - // Converts blank fields etc and preps the data. - var customfieldslst = "CustomFields"; - kfcustomfields = config.GetSection(customfieldslst).Get>(); - if (kfcustomfields == null) - { - kfcustomfields = new List(); - } - } + totalfieldsadded += manualresult.Item1; + totalfieldsadded += customresult.Item1; - //Adding metadata fields for the ID and the email of the requester from DigiCert. - List kfmanualfields = new List(); - var manualfieldslist = "ManualFields"; - kfmanualfields = config.GetSection(manualfieldslist).Get>(); - if (kfmanualfields == null) - { - kfmanualfields = new List(); - } - logger.LogDebug("Performed field conversion."); - - //Pulling list of existing metadata fields from Keyfactor for later comparison. - bool noexistingfields = true; - - var existingmetadataurl = keyfactorapilocation + "MetadataFields"; - var existingmetadataclient = new RestClient(); - existingmetadataclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); - var existingmetadatareq = new RestRequest(existingmetadataurl); - existingmetadatareq.AddHeader("Accept", "application/json"); - existingmetadatareq.AddHeader("x-keyfactor-api-version", "1"); - existingmetadatareq.AddHeader("x-keyfactor-requested-with", "APIClient"); - var existingmetadataresponse = existingmetadataclient.Execute(existingmetadatareq); - List existingmetadatalist = new List(); - if (existingmetadataresponse != null) + var allnewfields = manualresult.Item2.Concat(customresult.Item2).ToList(); + // Syncing Data from Keyfactor TO DigiCert + // Sync from DigiCert to Keyfactor must run at least once prior to this - only runs with custom fields + if (config_mode == "kftodc") + { + Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + _logger.Debug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + + var fullcustomdgfieldlist = new List(); + var newcustomfieldsfordg = new List(); + // Rebuild the list of metadata field names as they are on DigiCerts side. + + // This covers all of the custom fields on Digicerts side + foreach (var dgcustomfield in customdigicertmetadatafieldlist) { - //Fields exist - existingmetadatalist = JsonConvert.DeserializeObject>(existingmetadataresponse.Content); - noexistingfields = false; - } - Console.WriteLine("Pulled existing metadata fields from keyfactor."); - logger.LogDebug("Pulled existing metadata fields from Keyfactor."); - // Converting the read in fields into sendable lists - var convertedmanualfields = convertlisttokf(kfmanualfields, replacementcharacter); - var convertedcustomfields = convertlisttokf(kfcustomfields, replacementcharacter); + var localdigicertfieldinstance = new DigicertCustomFieldInstance(); + + localdigicertfieldinstance.label = dgcustomfield.label; + localdigicertfieldinstance.is_active = dgcustomfield.is_active; + localdigicertfieldinstance.data_type = dgcustomfield.data_type; + localdigicertfieldinstance.is_required = dgcustomfield.is_required; - int totalfieldsadded = 0; + foreach (var kffieldeq in kfcustomfields) + if (dgcustomfield.label == kffieldeq.DigicertFieldName) + localdigicertfieldinstance.kf_field_name = kffieldeq.DigicertFieldName; - //If all the fields are absent from Keyfactor, the fields are added. - var manualresult = AddFieldsToKeyfactor(convertedmanualfields, existingmetadatalist, noexistingfields, keyfactorusername, keyfactorpassword, keyfactorapilocation); - var customresult = AddFieldsToKeyfactor(convertedcustomfields, existingmetadatalist, noexistingfields, keyfactorusername, keyfactorpassword, keyfactorapilocation); + fullcustomdgfieldlist.Add(localdigicertfieldinstance); + } - totalfieldsadded += manualresult.Item1; - totalfieldsadded += customresult.Item1; - var allnewfields = manualresult.Item2.Concat(customresult.Item2).ToList(); - // Syncing Data from Keyfactor TO DigiCert - // Sync from DigiCert to Keyfactor must run at least once prior to this - only runs with custom fields - if (config_mode == "kftodc") + //This covers all of the new fields on Keyfactors side, including new ones - needs to have digicert ids for the new ones + foreach (var kfcustomfield in kfcustomfields) { - Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - logger.LogDebug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - - List fullcustomdgfieldlist = new List(); - List newcustomfieldsfordg = new List(); - // Rebuild the list of metadata field names as they are on DigiCerts side. - - // This covers all of the custom fields on Digicerts side - foreach (var dgcustomfield in customdigicertmetadatafieldlist) + var localdigicertfieldinstance = new DigicertCustomFieldInstance(); + localdigicertfieldinstance.label = kfcustomfield.DigicertFieldName; + localdigicertfieldinstance.is_active = true; + localdigicertfieldinstance.kf_field_name = kfcustomfield.KeyfactorMetadataFieldName; + if (kfcustomfield.KeyfactorDataType == "String") + localdigicertfieldinstance.data_type = "text"; + else if (kfcustomfield.KeyfactorDataType == "Int") + localdigicertfieldinstance.data_type = "int"; + else + localdigicertfieldinstance.data_type = "anything"; + localdigicertfieldinstance.is_required = false; + + if (!fullcustomdgfieldlist.Any(p => p.label == localdigicertfieldinstance.label)) { - DigicertCustomFieldInstance localdigicertfieldinstance = new DigicertCustomFieldInstance(); - - localdigicertfieldinstance.label = dgcustomfield.label; - localdigicertfieldinstance.is_active = dgcustomfield.is_active; - localdigicertfieldinstance.data_type = dgcustomfield.data_type; - localdigicertfieldinstance.is_required = dgcustomfield.is_required; - - foreach (var kffieldeq in kfcustomfields) - { - if (dgcustomfield.label == kffieldeq.DigicertFieldName) - { - localdigicertfieldinstance.kf_field_name = kffieldeq.DigicertFieldName; - } - } - fullcustomdgfieldlist.Add(localdigicertfieldinstance); + newcustomfieldsfordg.Add(localdigicertfieldinstance); } + } - - //This covers all of the new fields on Keyfactors side, including new ones - needs to have digicert ids for the new ones - foreach (var kfcustomfield in kfcustomfields) - { - DigicertCustomFieldInstance localdigicertfieldinstance = new DigicertCustomFieldInstance(); - localdigicertfieldinstance.label = kfcustomfield.DigicertFieldName; - localdigicertfieldinstance.is_active = true; - localdigicertfieldinstance.kf_field_name = kfcustomfield.KeyfactorMetadataFieldName; - if (kfcustomfield.KeyfactorDataType == "String") - { - localdigicertfieldinstance.data_type = "text"; - } - else if (kfcustomfield.KeyfactorDataType == "Int") - { - localdigicertfieldinstance.data_type = "int"; - } - else - { - localdigicertfieldinstance.data_type = "anything"; - } - localdigicertfieldinstance.is_required = false; - - if (!fullcustomdgfieldlist.Any(p => p.label == localdigicertfieldinstance.label)) - { - fullcustomdgfieldlist.Add(localdigicertfieldinstance); - newcustomfieldsfordg.Add(localdigicertfieldinstance); - } - } - - //Add fields that don't exist on DigiCert to Digicert - foreach (var newdgfield in newcustomfieldsfordg) - { - var digicertapilocation = "https://www.digicert.com/services/v2/account/metadata"; - var digicertnewfieldsclient = new RestClient(); - var digicertnewfieldsrequest = new RestRequest(digicertapilocation); - digicertnewfieldsrequest.AddHeader("Accept", "application/json"); - digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikeytopperm); - var serializedsyncfield = JsonConvert.SerializeObject(newdgfield); - digicertnewfieldsrequest.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); - var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); + //Add fields that don't exist on DigiCert to Digicert + _logger.Trace("Adding following fields to DigiCert: {0}", JsonConvert.SerializeObject(newcustomfieldsfordg)); + foreach (var newdgfield in newcustomfieldsfordg) + { + var digicertapilocation = "https://www.digicert.com/services/v2/account/metadata"; + var digicertnewfieldsclient = new RestClient(); + var digicertnewfieldsrequest = new RestRequest(digicertapilocation); + digicertnewfieldsrequest.AddHeader("Accept", "application/json"); + digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikeytopperm); + var serializedsyncfield = JsonConvert.SerializeObject(newdgfield); + digicertnewfieldsrequest.AddParameter("application/json", serializedsyncfield, + ParameterType.RequestBody); + var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); + } - } + // Grabbing the list again from digicert, populating ids for new ones + //Getting list of custom metadata fields on DigiCert + var updatedmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey); + foreach (var subitem in updatedmetadatafieldlist) + foreach (var fulllistitem in fullcustomdgfieldlist) + if (subitem.label == fulllistitem.label) + fulllistitem.id = subitem.id; + + var totalcertsprocessed = 0; + var numcertsdatauploaded = 0; + + // Pushing the data to DigiCert + var certlist2 = JsonConvert.DeserializeObject(rawresponse, + new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + foreach (var cert in certlist2) + { + Dictionary kfstoredmetadata = cert["Metadata"].ToObject>(); - // Grabbing the list again from digicert, populating ids for new ones - //Getting list of custom metadata fields on DigiCert - var updatedmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey); - foreach (var subitem in updatedmetadatafieldlist) - { - foreach (var fulllistitem in fullcustomdgfieldlist) - { - if (subitem.label == fulllistitem.label) - { - fulllistitem.id = subitem.id; - } - } - - } + var certhascustomfields = false; + foreach (var checkfield in fullcustomdgfieldlist) + if (kfstoredmetadata.ContainsKey(checkfield.kf_field_name)) + certhascustomfields = true; - var totalcertsprocessed = 0; - var numcertsdatauploaded = 0; - - // Pushing the data to DigiCert - var certlist2 = JsonConvert.DeserializeObject(rawresponse, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - foreach (var cert in certlist2) + if (certhascustomfields) { + var kfserialnumber = cert["SerialNumber"].ToString(); - Dictionary kfstoredmetadata = cert["Metadata"].ToObject>(); + var digicertnewlookupurl = "https://www.digicert.com/services/v2/order/certificate" + + "?filters[serial_number]=" + kfserialnumber; - bool certhascustomfields = false; - foreach (var checkfield in fullcustomdgfieldlist) - { - if (kfstoredmetadata.ContainsKey(checkfield.kf_field_name)) - { - certhascustomfields = true; - } - } + var newbodytemplate = new RootDigicertLookup(); + var newsearchcriterioninstance = new SearchCriterion(); + newbodytemplate.searchCriteriaList.Add(newsearchcriterioninstance); + var lookupnewrequest = new RestRequest(digicertnewlookupurl); + lookupnewrequest.AddHeader("Content-Type", "application/json"); + lookupnewrequest.AddHeader("X-DC-DEVKEY", digicertapikey); + var digicertnewlookupresponse = client.Execute(lookupnewrequest); + var newparseddigicertresponse = + JsonConvert.DeserializeObject(digicertnewlookupresponse.Content); - if (certhascustomfields){ - var kfserialnumber = cert["SerialNumber"].ToString(); - var digicertnewlookupurl = "https://www.digicert.com/services/v2/order/certificate" + "?filters[serial_number]=" + kfserialnumber; - - var newbodytemplate = new RootDigicertLookup(); - var newsearchcriterioninstance = new SearchCriterion(); - newbodytemplate.searchCriteriaList.Add(newsearchcriterioninstance); - var lookupnewrequest = new RestRequest(digicertnewlookupurl); - lookupnewrequest.AddHeader("Content-Type", "application/json"); - lookupnewrequest.AddHeader("X-DC-DEVKEY", digicertapikey); - var digicertnewlookupresponse = client.Execute(lookupnewrequest); - var newparseddigicertresponse = JsonConvert.DeserializeObject(digicertnewlookupresponse.Content); + if (newparseddigicertresponse["page"]["total"] != 0) + { + var newflatteneddigicertinstance = newparseddigicertresponse["orders"][0]; + var orderid = newflatteneddigicertinstance["id"].ToString(); + var digicertmetadataupdateapilocation = + "https://www.digicert.com/services/v2/order/certificate/" + orderid + "/custom-field"; + var digicertnewfieldsclient = new RestClient(); + var digicertnewfieldsrequest = new RestRequest(digicertmetadataupdateapilocation); + digicertnewfieldsrequest.AddHeader("Accept", "application/json"); + digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikey); - if (newparseddigicertresponse["page"]["total"] != 0) + foreach (var newfield in fullcustomdgfieldlist) { - var newflatteneddigicertinstance = newparseddigicertresponse["orders"][0]; - var orderid = newflatteneddigicertinstance["id"].ToString(); - - var digicertmetadataupdateapilocation = "https://www.digicert.com/services/v2/order/certificate/" + orderid + "/custom-field"; - var digicertnewfieldsclient = new RestClient(); - var digicertnewfieldsrequest = new RestRequest(digicertmetadataupdateapilocation); - digicertnewfieldsrequest.AddHeader("Accept", "application/json"); - digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikey); - - foreach (var newfield in fullcustomdgfieldlist) - { - string keyfactorfieldname = ""; - bool datauploaded = false; - //Lookup the keyfactor name for digicert fields - foreach (var sublookup in kfcustomfields) + var keyfactorfieldname = ""; + var datauploaded = false; + //Lookup the keyfactor name for digicert fields + foreach (var sublookup in kfcustomfields) + if (sublookup.DigicertFieldName == newfield.label) { - if (sublookup.DigicertFieldName == newfield.label) + var metadatapayload = new Dictionary(); + metadatapayload["metadata_id"] = newfield.id.ToString(); + if (kfstoredmetadata.ContainsKey(sublookup.KeyfactorMetadataFieldName)) { - Dictionary metadatapayload = new Dictionary(); - metadatapayload["metadata_id"] = newfield.id.ToString(); - if (kfstoredmetadata.ContainsKey(sublookup.KeyfactorMetadataFieldName)) - { - metadatapayload["value"] = kfstoredmetadata[sublookup.KeyfactorMetadataFieldName]; - var newserializedsyncfield = JsonConvert.SerializeObject(metadatapayload); - digicertnewfieldsrequest.AddParameter("application/json", newserializedsyncfield, ParameterType.RequestBody); - var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); - datauploaded = true; - } - + metadatapayload["value"] = + kfstoredmetadata[sublookup.KeyfactorMetadataFieldName]; + var newserializedsyncfield = JsonConvert.SerializeObject(metadatapayload); + digicertnewfieldsrequest.AddParameter("application/json", + newserializedsyncfield, ParameterType.RequestBody); + var digicertresponsenewfields = + digicertnewfieldsclient.Post(digicertnewfieldsrequest); + datauploaded = true; } } - } - numcertsdatauploaded += 1; } - } - totalcertsprocessed += 1; - } - Console.WriteLine($"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); - Console.WriteLine($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); - logger.LogDebug($"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); - logger.LogDebug($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); - - } - // Syncing Data from DigiCert TO Keyfactor - if (config_mode == "dctokf") - { - Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - logger.LogDebug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - //Each cert that is DigiCert in origin in Keyfactor is looked up on DigiCert via serial number, - //and the metadata contents from those fields are stored. - var digicertlookupclient = new RestClient(); - List digicertcertificates = new List(); - foreach (var certinstance in certlist) - { - var digicertlookupurl = "https://www.digicert.com/services/v2/order/certificate/"; - - var bodytemplate = new RootDigicertLookup(); - var searchcriterioninstance = new SearchCriterion(); - bodytemplate.searchCriteriaList.Add(searchcriterioninstance); - - digicertlookupurl = digicertlookupurl + certinstance.SerialNumber; - var lookuprequest = new RestRequest(digicertlookupurl); - lookuprequest.AddHeader("Content-Type", "application/json"); - lookuprequest.AddHeader("X-DC-DEVKEY", digicertapikey); - var digicertlookupresponse = client.Execute(lookuprequest); - var parseddigicertresponse = JsonConvert.DeserializeObject(digicertlookupresponse.Content); - if (parseddigicertresponse.certificate != null) - { - var flatteneddigicertinstance = ClassConverter(parseddigicertresponse); - digicertcertificates.Add(parseddigicertresponse); + numcertsdatauploaded += 1; } } - Console.WriteLine("Pulled DigiCert matching DigiCert cert data."); - logger.LogDebug("Pulled DigiCert matching DigiCert cert data."); - - - int certcounttracker = 0; - foreach (var digicertcertinstance in digicertcertificates) - { - var finalsyncclient = new RestClient(); - finalsyncclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); - var finalsyncurl = keyfactorapilocation + "Certificates/Metadata"; - //Find matching certificate via Keyfactor ID - var query = from kfcertlocal in certlist - where kfcertlocal.SerialNumber == - digicertcertinstance.certificate.serial_number.ToUpper() - select kfcertlocal; - var certificateid = query.FirstOrDefault().Id; + totalcertsprocessed += 1; + } + Console.WriteLine( + $"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); + Console.WriteLine($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); + _logger.Debug( + $"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); + _logger.Debug($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); + } - var payloadforkf = new KeyfactorMetadataQuery(); - payloadforkf.Id = certificateid; + // Syncing Data from DigiCert TO Keyfactor + if (config_mode == "dctokf") + { + Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + _logger.Debug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + //Each cert that is DigiCert in origin in Keyfactor is looked up on DigiCert via serial number, + //and the metadata contents from those fields are stored. + var digicertlookupclient = new RestClient(); + var digicertcertificates = new List(); + foreach (var certinstance in certlist) + { + var digicertlookupurl = "https://www.digicert.com/services/v2/order/certificate/"; - if (digicertcertinstance.custom_fields != null) - { - // Getting custom metadata field values - foreach (var metadatafieldinstance in digicertcertinstance.custom_fields) - { - if (importallcustomdigicertfields == true) - { - // Using autoimport and thus using autorename - payloadforkf.Metadata[ReplaceAllWhiteSpaces(metadatafieldinstance.label, replacementcharacter)] = metadatafieldinstance.value; - } - else - { - //Using custom names - var metadatanamequery = from customfieldinstance in kfcustomfields - where customfieldinstance.DigicertFieldName == - metadatafieldinstance.label - select customfieldinstance; - if (metadatanamequery.FirstOrDefault() != null) - { - payloadforkf.Metadata[metadatanamequery.FirstOrDefault().DigicertFieldName] = metadatafieldinstance.value; - } + var bodytemplate = new RootDigicertLookup(); + var searchcriterioninstance = new SearchCriterion(); + bodytemplate.searchCriteriaList.Add(searchcriterioninstance); - } + digicertlookupurl = digicertlookupurl + certinstance.SerialNumber; + var lookuprequest = new RestRequest(digicertlookupurl); + lookuprequest.AddHeader("Content-Type", "application/json"); + lookuprequest.AddHeader("X-DC-DEVKEY", digicertapikey); + var digicertlookupresponse = client.Execute(lookuprequest); + var certcontent = JsonConvert.DeserializeObject(digicertlookupresponse.Content, + new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - } - } + if (certcontent["certificate"] != null) + { + digicertcertificates.Add(certcontent); + _logger.Trace("Pulled and storing following cert from digicert: {0}", + digicertlookupresponse.Content); + } + } - var flatteneddigicertinstance = ClassConverter(digicertcertinstance); + Console.WriteLine("Pulled DigiCert matching DigiCert cert data."); + _logger.Debug("Pulled DigiCert matching DigiCert cert data."); - //Getting manually selected metadata field values (not custom in DigiCert) - foreach (var manualinstance in kfmanualfields) - { - string[] access = manualinstance.DigicertFieldName.Split("."); - List keys = access.ToList(); - Dictionary recursionresult = recursiveopener(flatteneddigicertinstance, keys, keys.Count); - object value = new object(); - if (recursionresult != null) + var certcounttracker = 0; + foreach (var digicertcertinstance in digicertcertificates) + { + var finalsyncclient = new RestClient(); + finalsyncclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); + var finalsyncurl = keyfactorapilocation + "Certificates/Metadata"; + //Find matching certificate via Keyfactor ID + var test = digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper(); + var query = from kfcertlocal in certlist + where kfcertlocal.SerialNumber == + digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper() + select kfcertlocal; + var certificateid = query.FirstOrDefault().Id; + + + var payloadforkf = new KeyfactorMetadataQuery(); + payloadforkf.Id = certificateid; + + if (digicertcertinstance["custom_fields"] != null) + // Getting custom metadata field values + foreach (var metadatafieldinstance in digicertcertinstance["custom_fields"]) + if (importallcustomdigicertfields) { - value = recursionresult.First().Value; + // Using autoimport and thus using autorename + var metadatanamefield = ReplaceAllWhiteSpaces(metadatafieldinstance["label"].ToString(), + replacementcharacter); + payloadforkf.Metadata[metadatanamefield] = metadatafieldinstance["value"]; } else { - value = ""; + //Using custom names + var metadatanamequery = from customfieldinstance in kfcustomfields + where customfieldinstance.DigicertFieldName == + metadatafieldinstance["label"] + select customfieldinstance; + if (metadatanamequery.FirstOrDefault() != null) + payloadforkf.Metadata[metadatanamequery.FirstOrDefault().DigicertFieldName] = + metadatafieldinstance["value"]; } - payloadforkf.Metadata[manualinstance.KeyfactorMetadataFieldName] = value; - } - payloadforkf.Metadata["DigicertID"] = digicertcertinstance.id.ToString(); - //Sending the payload off to Keyfactor for the update - var finalsyncreq = new RestRequest(finalsyncurl); - finalsyncreq.AddHeader("Content-Type", "application/json"); - finalsyncreq.AddHeader("x-keyfactor-api-version", "1"); - finalsyncreq.AddHeader("x-keyfactor-requested-with", "APIClient"); - var serializedsyncfield = JsonConvert.SerializeObject(payloadforkf); - finalsyncreq.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); - finalsyncclient.Put(finalsyncreq); - ++certcounttracker; - } - Console.WriteLine($"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); - logger.LogDebug($"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); + var flattenedcert = Flatten(digicertcertinstance); + //Getting manually selected metadata field values (not custom in DigiCert) + foreach (var manualinstance in kfmanualfields) + if (flattenedcert[manualinstance.DigicertFieldName] != null) + payloadforkf.Metadata[manualinstance.KeyfactorMetadataFieldName] = + flattenedcert[manualinstance.DigicertFieldName].ToString(); + //Sending the payload off to Keyfactor for the update + var finalsyncreq = new RestRequest(finalsyncurl); + finalsyncreq.AddHeader("Content-Type", "application/json"); + finalsyncreq.AddHeader("x-keyfactor-api-version", "1"); + finalsyncreq.AddHeader("x-keyfactor-requested-with", "APIClient"); + var serializedsyncfield = JsonConvert.SerializeObject(payloadforkf); + _logger.Trace("Sending Metadata update to KF for cert ID {0}, metadata update: {1}", + payloadforkf.Id.ToString(), serializedsyncfield); + + finalsyncreq.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); + finalsyncclient.Put(finalsyncreq); + ++certcounttracker; } + + Console.WriteLine( + $"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); + _logger.Debug( + $"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); } } -} - - +} \ No newline at end of file diff --git a/digicert-metadata-sync/Models/DigicertCertInstance.cs b/digicert-metadata-sync/Models/DigicertCertInstance.cs index 879e103..e96dc9d 100644 --- a/digicert-metadata-sync/Models/DigicertCertInstance.cs +++ b/digicert-metadata-sync/Models/DigicertCertInstance.cs @@ -13,7 +13,8 @@ // limitations under the License. namespace DigicertMetadataSync; -partial class DigicertSync + +internal partial class DigicertSync { public class DigicertCert { diff --git a/digicert-metadata-sync/Models/DigicertCustomField.cs b/digicert-metadata-sync/Models/DigicertCustomField.cs index a622ce7..3e2def7 100644 --- a/digicert-metadata-sync/Models/DigicertCustomField.cs +++ b/digicert-metadata-sync/Models/DigicertCustomField.cs @@ -14,9 +14,8 @@ namespace DigicertMetadataSync; -partial class DigicertSync +internal partial class DigicertSync { - public class DigicertCustomFieldInstance { public int id { get; set; } = 999999999; @@ -30,7 +29,6 @@ public class DigicertCustomFieldInstance public class DigicertMetadataUpdateInstance { public int metadata_id { get; set; } = 999999999; - public string value { get; set; } = "false"; + public string value { get; set; } = "false"; } - } \ No newline at end of file diff --git a/digicert-metadata-sync/Models/DigicertLookup.cs b/digicert-metadata-sync/Models/DigicertLookup.cs index d3f01c4..365c43e 100644 --- a/digicert-metadata-sync/Models/DigicertLookup.cs +++ b/digicert-metadata-sync/Models/DigicertLookup.cs @@ -14,19 +14,19 @@ namespace DigicertMetadataSync; -partial class DigicertSync +internal partial class DigicertSync { public class RootDigicertLookup { - public List searchCriteriaList = new List(); + public List searchCriteriaList = new(); public string accountId { get; set; } = "x"; } public class SearchCriterion { + public List value = new(); public string key { get; set; } = "serialNum"; public string operation { get; set; } = "EQUALS"; - public List value = new(); } } \ No newline at end of file diff --git a/digicert-metadata-sync/Models/InternalClasses.cs b/digicert-metadata-sync/Models/InternalClasses.cs index 0bdd25a..1cf7082 100644 --- a/digicert-metadata-sync/Models/InternalClasses.cs +++ b/digicert-metadata-sync/Models/InternalClasses.cs @@ -14,7 +14,7 @@ namespace DigicertMetadataSync; -partial class DigicertSync +internal partial class DigicertSync { public class CustomDigicertMetadataInstance { diff --git a/digicert-metadata-sync/Models/KeyfactorCertInstance.cs b/digicert-metadata-sync/Models/KeyfactorCertInstance.cs index 11e3670..73c24e4 100644 --- a/digicert-metadata-sync/Models/KeyfactorCertInstance.cs +++ b/digicert-metadata-sync/Models/KeyfactorCertInstance.cs @@ -14,7 +14,7 @@ namespace DigicertMetadataSync; -partial class DigicertSync +internal partial class DigicertSync { public class KeyfactorCert { @@ -66,7 +66,6 @@ public class KeyfactorCert public class Metadata { - } public class Detailedkeyusage diff --git a/digicert-metadata-sync/Models/KeyfactorMetadataInstance.cs b/digicert-metadata-sync/Models/KeyfactorMetadataInstance.cs index 0ef8567..043c9ca 100644 --- a/digicert-metadata-sync/Models/KeyfactorMetadataInstance.cs +++ b/digicert-metadata-sync/Models/KeyfactorMetadataInstance.cs @@ -14,7 +14,7 @@ namespace DigicertMetadataSync; -partial class DigicertSync +internal partial class DigicertSync { //This stores all of the data keyfactor API returns when asked for metadata field details. public class KeyfactorMetadataInstance diff --git a/digicert-metadata-sync/Models/KeyfactorMetadataQuery.cs b/digicert-metadata-sync/Models/KeyfactorMetadataQuery.cs index 5dcb24a..4cc8dde 100644 --- a/digicert-metadata-sync/Models/KeyfactorMetadataQuery.cs +++ b/digicert-metadata-sync/Models/KeyfactorMetadataQuery.cs @@ -14,11 +14,11 @@ namespace DigicertMetadataSync; -partial class DigicertSync +internal partial class DigicertSync { public class KeyfactorMetadataQuery { + public Dictionary Metadata = new(); public int Id { get; set; } - public Dictionary Metadata = new Dictionary(); } } \ No newline at end of file diff --git a/digicert-metadata-sync/NLog.config b/digicert-metadata-sync/NLog.config new file mode 100644 index 0000000..1ff0000 --- /dev/null +++ b/digicert-metadata-sync/NLog.config @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/readme_source.md b/readme_source.md index cdb8bcf..5fafadd 100644 --- a/readme_source.md +++ b/readme_source.md @@ -1,27 +1,16 @@ - - ## Overview -This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. The latest update allows for syncronization of custom field contents from Keyfactor to DigiCert. New fields are created in Keyfactor and DigiCert to accomodate for this. +This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. ## Installation and Usage -The tool comes as a Windows executable. The tool performs synchronization each time its run. For the tool to run automatically, it needs to be added as a scheduled process using Windows. The advised interval for running it is once per week. The files App.config and manualfields.json need to be present in the same directory as the tool for it to run correctly. The specific location from which the tool is ran does not matter, but it needs to have access to both the Keyfactor API endpoint as well as Digicert, and appropriate permissions for access to the configuration files. +The tool comes as a Windows executable. The tool performs synchronization each time its run. For the tool to run automatically, it needs to be added as a scheduled process using Windows. The advised interval for running it is once per week. The files DigicertMetadataSync.dll.config and manualfields.json need to be present in the same directory as the tool for it to run correctly. The specific location from which the tool is ran does not matter, but it needs to have access to both the Keyfactor API endpoint as well as Digicert, and appropriate permissions for access to the configuration files. An explanation for the settings found in these files is given below. -## Command Line Arguments -One of these two arguments needs to be used for the tool to run. -- "kftodc" -Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. Example: ```.\DigicertMetadataSync.exe kftodc``` -- "dctokf" -Syncronizes the contents of both custom and non-custom fields from DigiCert to Keyfactor. The fields are listed in manualfields.json, and are created if necessary. -Example: ```.\DigicertMetadataSync.exe dctokf``` ## Settings The settings currently present in these files are shown as an example and need to be configured for your specific situation. -### app.config settings +### DigicertMetadataSync.dll.config settings - DigicertAPIKey -Standard DigiCert API access key. -- DigicertAPIKeyTopPerm -DigiCert API access key with restrictions set to "None" - required for sync from Keyfactor to DigiCert. +Standard DigiCert API access key - KeyfactorDomainAndUser Same credential as used when logging into Keyfactor Command. A different set of credentials can be used provided they have adequate access permissions. - KeyfactorPassword @@ -65,3 +54,5 @@ String to be input into Keyfactor as the metadata field hint. - KeyfactorAllowAPI Allows API management of this metadata field in Keyfactor. Should be set to true for continuous synchronization with this tool. +### Logging +Logging functionality can be configured via entering either "Debug" or "Trace" into the value of `` in NLog.config. From 8d7c8c8d62f08a49b4a47eb85799ffe148bcdf04 Mon Sep 17 00:00:00 2001 From: Mark Kachkaev <37276742+mkachk@users.noreply.github.com> Date: Mon, 28 Aug 2023 11:37:45 -0400 Subject: [PATCH 06/12] Release 2.0.3. (#19) * Added a switch to control syncing of fields deactivated in DigiCert. * Updated readme_source.md with new config section. Fixes ab#47725, ab#46882 --- CHANGELOG.md | 4 ++++ README.md | 5 ++-- digicert-metadata-sync/App.config | 1 + .../GrabCustomFieldsFromDigiCert.cs | 10 ++++++-- digicert-metadata-sync/MetadataSync.cs | 24 ++++++++++--------- readme_source.md | 2 ++ 6 files changed, 30 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5552b98..796d4f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +Version 2.0.3 + + Added a setting to enable or disable syncing deactivated custom fields from DigiCert. + Version 2.0.2 Fixed issue with additional_emails field not syncing. diff --git a/README.md b/README.md index 4ca324c..af5548c 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,6 @@ A tool to automatically synchronize metadata fields and their content from DigiC - - ## Support for Digicert Metadata Sync Digicert Metadata Sync is open source and there is **no SLA** for this tool/library/client. Keyfactor will address issues as resources become available. Keyfactor customers may request escalation by opening up a support ticket through their Keyfactor representative. @@ -16,7 +14,6 @@ Digicert Metadata Sync is open source and there is **no SLA** for this tool/libr - ## Overview This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. @@ -44,6 +41,8 @@ This should include the common prefix all DigiCert certs have in your Keyfactor This setting enables the tool to import all of the custom metadata fields included in DigiCert and sync all of their data. - ReplaceDigicertWhiteSpaceCharacterInName In case the ImportAllCustomDigicertFields setting is used, this is necessary to for metadata field label conversion. DigiCert supports spaces in labels and Keyfactor does not, so this replaces the spaces in the name with your character sequence of choice. +- ImportDataForDeactivatedDigiCertFields +If this is enabled, custom metadata fields that were deactivated in DigiCert will also be synced, and the data stored in these fields in certificates will be too. ### manualfields.json settings This file is used to specify which metadata fields should be synced up. diff --git a/digicert-metadata-sync/App.config b/digicert-metadata-sync/App.config index 301d1d0..c17c000 100644 --- a/digicert-metadata-sync/App.config +++ b/digicert-metadata-sync/App.config @@ -10,5 +10,6 @@ + \ No newline at end of file diff --git a/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs b/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs index 0236336..8795602 100644 --- a/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs +++ b/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs @@ -15,6 +15,7 @@ using Keyfactor.Logging; using Microsoft.Extensions.Logging; using Newtonsoft.Json; +using NLog.Time; using RestSharp; using RestSharp.Authenticators; @@ -24,8 +25,9 @@ namespace DigicertMetadataSync; // It will only add new fields. partial class DigicertSync { - public static List GrabCustomFieldsFromDigiCert(string apikey) + public static List GrabCustomFieldsFromDigiCert(string apikey, bool importdeactivated) { + ILogger logger = LogHandler.GetClassLogger(); var digicertclient = new RestClient(); var customfieldsretrieval = "https://www.digicert.com/services/v2/account/metadata"; var digicertrequest = new RestRequest(customfieldsretrieval); @@ -36,8 +38,12 @@ public static List GrabCustomFieldsFromDigiCert( int lengthofresponse = trimmeddigicertresponse.Length; trimmeddigicertresponse = trimmeddigicertresponse.Remove(lengthofresponse - 1, 1); var fieldlist = JsonConvert.DeserializeObject>(trimmeddigicertresponse); + if (importdeactivated == false) + { + fieldlist.RemoveAll(unit => unit.is_active == false); + } Console.WriteLine("Obtained custom fields from DigiCert."); - _logger.Debug("Obtained custom fields from DigiCert."); + logger.LogDebug("Obtained custom fields from DigiCert."); return fieldlist; } } \ No newline at end of file diff --git a/digicert-metadata-sync/MetadataSync.cs b/digicert-metadata-sync/MetadataSync.cs index 380c05b..26ad9fa 100644 --- a/digicert-metadata-sync/MetadataSync.cs +++ b/digicert-metadata-sync/MetadataSync.cs @@ -11,6 +11,7 @@ using RestSharp; using RestSharp.Authenticators; using ConfigurationManager = System.Configuration.ConfigurationManager; +//using Keyfactor.Logging; namespace DigicertMetadataSync; @@ -26,6 +27,7 @@ public static void Main(string[] args) var digicertapikeytopperm = ConfigurationManager.AppSettings.Get("DigicertAPIKeyTopPerm"); var keyfactorusername = ConfigurationManager.AppSettings.Get("KeyfactorDomainAndUser"); var keyfactorpassword = ConfigurationManager.AppSettings.Get("KeyfactorPassword"); + var importdeactivated = Convert.ToBoolean(ConfigurationManager.AppSettings.Get("ImportDataForDeactivatedDigiCertFields")); var replacementcharacter = ConfigurationManager.AppSettings.Get("ReplaceDigicertWhiteSpaceCharacterInName"); var importallcustomdigicertfields = Convert.ToBoolean(ConfigurationManager.AppSettings.Get("ImportAllCustomDigicertFields")); @@ -75,7 +77,7 @@ public static void Main(string[] args) _logger.Debug("Got list of custom fields from Keyfactor."); //Getting list of custom metadata fields on DigiCert - var customdigicertmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey); + var customdigicertmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey, importdeactivated); //Convert DigiCert custom fields to Keyfactor appropriate ones //This depends on whether the setting to import all fields was enabled or not @@ -253,11 +255,11 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. // Grabbing the list again from digicert, populating ids for new ones //Getting list of custom metadata fields on DigiCert - var updatedmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey); + var updatedmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey, importdeactivated); foreach (var subitem in updatedmetadatafieldlist) - foreach (var fulllistitem in fullcustomdgfieldlist) - if (subitem.label == fulllistitem.label) - fulllistitem.id = subitem.id; + foreach (var fulllistitem in fullcustomdgfieldlist) + if (subitem.label == fulllistitem.label) + fulllistitem.id = subitem.id; var totalcertsprocessed = 0; var numcertsdatauploaded = 0; @@ -389,9 +391,9 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. //Find matching certificate via Keyfactor ID var test = digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper(); var query = from kfcertlocal in certlist - where kfcertlocal.SerialNumber == - digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper() - select kfcertlocal; + where kfcertlocal.SerialNumber == + digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper() + select kfcertlocal; var certificateid = query.FirstOrDefault().Id; @@ -412,9 +414,9 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. { //Using custom names var metadatanamequery = from customfieldinstance in kfcustomfields - where customfieldinstance.DigicertFieldName == - metadatafieldinstance["label"] - select customfieldinstance; + where customfieldinstance.DigicertFieldName == + metadatafieldinstance["label"] + select customfieldinstance; if (metadatanamequery.FirstOrDefault() != null) payloadforkf.Metadata[metadatanamequery.FirstOrDefault().DigicertFieldName] = metadatafieldinstance["value"]; diff --git a/readme_source.md b/readme_source.md index 5fafadd..ddb76b2 100644 --- a/readme_source.md +++ b/readme_source.md @@ -25,6 +25,8 @@ This should include the common prefix all DigiCert certs have in your Keyfactor This setting enables the tool to import all of the custom metadata fields included in DigiCert and sync all of their data. - ReplaceDigicertWhiteSpaceCharacterInName In case the ImportAllCustomDigicertFields setting is used, this is necessary to for metadata field label conversion. DigiCert supports spaces in labels and Keyfactor does not, so this replaces the spaces in the name with your character sequence of choice. +- ImportDataForDeactivatedDigiCertFields +If this is enabled, custom metadata fields that were deactivated in DigiCert will also be synced, and the data stored in these fields in certificates will be too. ### manualfields.json settings This file is used to specify which metadata fields should be synced up. From 92bd62d059f127a7d88058136c4477268c75282d Mon Sep 17 00:00:00 2001 From: Keyfactor Date: Mon, 28 Aug 2023 17:06:41 +0000 Subject: [PATCH 07/12] Update generated README --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b719524..3ccc863 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ Digicert Metadata Sync is open source and there is **no SLA** for this tool/libr + + ## Overview This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. The latest update allows for syncronization of custom field contents from Keyfactor to DigiCert. New fields are created in Keyfactor and DigiCert to accomodate for this. @@ -83,4 +85,4 @@ Allows API management of this metadata field in Keyfactor. Should be set to true ### Logging Logging functionality can be configured via entering either "Debug" or "Trace" into the value of `` in NLog.config. -======= + From bb35b740f9784ac8f347ba49199e68c0947fbf40 Mon Sep 17 00:00:00 2001 From: Mark Kachkaev Date: Mon, 16 Oct 2023 09:11:34 -0400 Subject: [PATCH 08/12] Added: automatic banned character recognition, paginated processing - as fixes for two bugs --- digicert-metadata-sync/BannedCharacters.cs | 68 ++ digicert-metadata-sync/Helpers.cs | 21 +- digicert-metadata-sync/MetadataSync.cs | 624 ++++++++++-------- digicert-metadata-sync/Models/CharDBItem.cs | 26 + .../Models/DigicertCustomField.cs | 6 +- .../Models/DigicertLookup.cs | 6 +- .../Models/InternalClasses.cs | 3 +- .../Models/KeyfactorCertInstance.cs | 2 +- digicert-metadata-sync/replacechar.json | 1 + 9 files changed, 476 insertions(+), 281 deletions(-) create mode 100644 digicert-metadata-sync/BannedCharacters.cs create mode 100644 digicert-metadata-sync/Models/CharDBItem.cs create mode 100644 digicert-metadata-sync/replacechar.json diff --git a/digicert-metadata-sync/BannedCharacters.cs b/digicert-metadata-sync/BannedCharacters.cs new file mode 100644 index 0000000..74e33fe --- /dev/null +++ b/digicert-metadata-sync/BannedCharacters.cs @@ -0,0 +1,68 @@ +// Copyright 2021 Keyfactor +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text.RegularExpressions; +using Newtonsoft.Json.Linq; + +namespace DigicertMetadataSync; + +internal partial class DigicertSync +{ + public static List BannedCharacterParse(string input) + { + string pattern = "[a-zA-Z0-9-_]"; + + List bannedChars = new List(); + + foreach (char c in input) + { + if (!Regex.IsMatch(c.ToString(), pattern)) + { + CharDBItem localitem = new CharDBItem(); + localitem.character = c.ToString(); + localitem.replacementcharacter = "null"; + bannedChars.Add(localitem); + } + } + + if (bannedChars.Count > 0) + { + Console.WriteLine("The field name " + input + " contains the following invalid characters: " + + string.Join("", bannedChars.Select(item => item.character))); + } + else + { + Console.WriteLine("The field name " + input + " is valid."); + } + + return bannedChars; + } + + public static void CheckForChars(List input, List allBannedChars, bool restartandconfigrequired) + { + foreach (var dgfield in input) + { + List newChars = BannedCharacterParse(dgfield.DigicertFieldName); + foreach (var newchar in newChars) + { + bool exists = allBannedChars.Any(allcharchar => allcharchar.character == newchar.character); + if (!exists) + { + allBannedChars.Add(newchar); + restartandconfigrequired = true; + } + } + } + } +} diff --git a/digicert-metadata-sync/Helpers.cs b/digicert-metadata-sync/Helpers.cs index 6dd9d45..dfd14b8 100644 --- a/digicert-metadata-sync/Helpers.cs +++ b/digicert-metadata-sync/Helpers.cs @@ -1,4 +1,4 @@ -// Copyright 2021 Keyfactor +// Copyright 2021 Keyfactor // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +using System.Collections.Generic; using System.Text.RegularExpressions; using Newtonsoft.Json.Linq; @@ -54,9 +55,13 @@ public static Dictionary ClassConverter(object obj) return null; } - public static string ReplaceAllWhiteSpaces(string str, string replacement) + public static string ReplaceAllBannedCharacters(string input, ListallBannedChars) { - return Regex.Replace(str, @"\s+", "_-_"); + foreach (CharDBItem item in allBannedChars) + { + input = input.Replace(item.character, item.replacementcharacter); + } + return input; } public static bool CheckMode(string mode) @@ -65,17 +70,16 @@ public static bool CheckMode(string mode) return false; } - private static List convertlisttokf(List inputlist, - string replacementcharacter) + private static List convertlisttokf(List inputlist, List allBannedChars, bool importallcustomfields) { var formattedlist = new List(); if (inputlist.Count != 0) foreach (var input in inputlist) { var formatinstance = new KeyfactorMetadataInstanceSendoff(); - if (input.KeyfactorMetadataFieldName == null || input.KeyfactorMetadataFieldName == "") - //If name is emtpy, use autocomplete. - formatinstance.Name = ReplaceAllWhiteSpaces(input.DigicertFieldName, replacementcharacter); + if (input.KeyfactorMetadataFieldName == null || input.KeyfactorMetadataFieldName == "" || input.FieldType == "Custom") + //If name is empty, clean up the characters. + formatinstance.Name = ReplaceAllBannedCharacters(input.DigicertFieldName, allBannedChars); else //Use user input preferred name. formatinstance.Name = input.KeyfactorMetadataFieldName; @@ -86,6 +90,7 @@ private static List convertlisttokf(List>(rawresponse, - new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - Console.WriteLine("Got DigiCert issued certs from keyfactor"); - _logger.Debug("Got DigiCert issued certs from keyfactor"); //Getting list of custom metadata fields from Keyfactor var getmetadalistkf = keyfactorapilocation + "MetadataFields"; @@ -85,6 +78,7 @@ public static void Main(string[] args) var config = new ConfigurationBuilder().SetBasePath(AppDomain.CurrentDomain.BaseDirectory) .AddJsonFile("manualfields.json").Build(); var kfcustomfields = new List(); + if (importallcustomdigicertfields) { _logger.Debug("Loading custom fields using autofill"); @@ -105,8 +99,7 @@ WHITESPACE MUST BE REMOVED FROM THE NAME. CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. */ localkffieldinstance.DigicertFieldName = customdigicertmetadatafieldlist[i].label; - localkffieldinstance.KeyfactorMetadataFieldName = - ReplaceAllWhiteSpaces(customdigicertmetadatafieldlist[i].label, replacementcharacter); + localkffieldinstance.KeyfactorMetadataFieldName = customdigicertmetadatafieldlist[i].label; _logger.Debug("DC field name {0} becomes {1} in Keyfactor", localkffieldinstance.DigicertFieldName, localkffieldinstance.KeyfactorMetadataFieldName); } @@ -137,12 +130,21 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. if (kfcustomfields == null) kfcustomfields = new List(); _logger.Debug("Loading custom fields using json, no autofill/conversion"); } + foreach (var item in kfcustomfields) + { + item.FieldType = "Custom"; + } + //Adding metadata fields for the ID and the email of the requester from DigiCert. var kfmanualfields = new List(); var manualfieldslist = "ManualFields"; kfmanualfields = config.GetSection(manualfieldslist).Get>(); if (kfmanualfields == null) kfmanualfields = new List(); + foreach (var item in kfmanualfields) + { + item.FieldType = "Manual"; + } _logger.Debug("Performed field conversion."); //Pulling list of existing metadata fields from Keyfactor for later comparison. @@ -167,9 +169,52 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. Console.WriteLine("Pulled existing metadata fields from keyfactor."); _logger.Debug("Pulled existing metadata fields from Keyfactor."); - // Converting the read in fields into sendable lists - var convertedmanualfields = convertlisttokf(kfmanualfields, replacementcharacter); - var convertedcustomfields = convertlisttokf(kfcustomfields, replacementcharacter); + + + + // Carrying out the persistent banned character database check + + // Loading up the character database + string currentDirectory = Directory.GetCurrentDirectory(); + + string filePath = Path.Combine(currentDirectory, "replacechar.json"); + + bool restartandconfigrequired = false; + + List allBannedChars = JsonConvert.DeserializeObject>(File.ReadAllText(filePath)); + + if (importallcustomdigicertfields) + { + CheckForChars(kfmanualfields, allBannedChars, restartandconfigrequired); + CheckForChars(kfcustomfields, allBannedChars, restartandconfigrequired); + + string formattedjsonchars = JsonConvert.SerializeObject(allBannedChars, Formatting.Indented); + File.WriteAllText(filePath, formattedjsonchars); + + foreach (var badchar in allBannedChars) + { + if (badchar.replacementcharacter == "null") + { + restartandconfigrequired = true; + break; + } + } + + if (restartandconfigrequired) + { + _logger.Trace("Please replace \"null\" with your desired replacement characters in replacechar.json and re-run the tool! Only alphanumerics, \"-\" and \"_\" are allowed"); + Console.WriteLine("Please replace \"null\" with your desired replacement characters in replacechar.json and re-run the tool! Only alphanumerics, \"-\" and \"_\" are allowed"); + Environment.Exit(0); + } + + + } + + // Converting the read in fields into sendable lists + var convertedmanualfields = convertlisttokf(kfmanualfields, allBannedChars, importallcustomdigicertfields); + var convertedcustomfields = convertlisttokf(kfcustomfields, allBannedChars, importallcustomdigicertfields); + + _logger.Trace("Sending following manual fields to KF: {0}", JsonConvert.SerializeObject(convertedmanualfields)); var totalfieldsadded = 0; @@ -186,311 +231,358 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. totalfieldsadded += customresult.Item1; var allnewfields = manualresult.Item2.Concat(customresult.Item2).ToList(); + + + //Processing this batch + Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + _logger.Debug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + // Syncing Data from Keyfactor TO DigiCert // Sync from DigiCert to Keyfactor must run at least once prior to this - only runs with custom fields if (config_mode == "kftodc") { - Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - _logger.Debug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - - var fullcustomdgfieldlist = new List(); - var newcustomfieldsfordg = new List(); - // Rebuild the list of metadata field names as they are on DigiCerts side. - - // This covers all of the custom fields on Digicerts side - foreach (var dgcustomfield in customdigicertmetadatafieldlist) + // Initialize variable to keep track of items downloaded so far + int certsdownloaded = 0; + var certcounttracker = 0; + var totalcertsprocessed = 0; + var numcertsdatauploaded = 0; + for (int batchnum = 0; batchnum < numberOfBatches; batchnum++) { - var localdigicertfieldinstance = new DigicertCustomFieldInstance(); + // Check if reaching the arbitrary limit + if (certsdownloaded + batchsize > returnlimitint) + { + Console.WriteLine($"Stopped downloading at the configured limit of {returnlimitint} items."); + _logger.Debug($"Stopped downloading at the configured limit of {returnlimitint} items."); + break; + } - localdigicertfieldinstance.label = dgcustomfield.label; - localdigicertfieldinstance.is_active = dgcustomfield.is_active; - localdigicertfieldinstance.data_type = dgcustomfield.data_type; - localdigicertfieldinstance.is_required = dgcustomfield.is_required; - foreach (var kffieldeq in kfcustomfields) - if (dgcustomfield.label == kffieldeq.DigicertFieldName) - localdigicertfieldinstance.kf_field_name = kffieldeq.DigicertFieldName; + var fullcustomdgfieldlist = new List(); + var newcustomfieldsfordg = new List(); + + // Download the items in this batch + var digicertlookup = keyfactorapilocation + "Certificates?pq.queryString=IssuerDN%20-contains%20%22" + + digicertIssuerQueryterm + "%22&pq.returnLimit=" + batchsize.ToString() + + "&includeMetadata=true" + "&pq.pageReturned=" + batchnum.ToString(); + var request = new RestRequest(digicertlookup); + request.AddHeader("Accept", "application/json"); + request.AddHeader("x-keyfactor-api-version", "1"); + request.AddHeader("x-keyfactor-requested-with", "APIClient"); + var response = client.Execute(request); + var rawresponse = response.Content; + var certlist = JsonConvert.DeserializeObject>(rawresponse, + new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + Console.WriteLine("Got DigiCert issued certs from keyfactor"); + _logger.Debug("Got DigiCert issued certs from keyfactor"); - fullcustomdgfieldlist.Add(localdigicertfieldinstance); - } + // Rebuild the list of metadata field names as they are on DigiCerts side. + // This covers all of the custom fields on Digicerts side + foreach (var dgcustomfield in customdigicertmetadatafieldlist) + { + var localdigicertfieldinstance = new DigicertCustomFieldInstance(); - //This covers all of the new fields on Keyfactors side, including new ones - needs to have digicert ids for the new ones - foreach (var kfcustomfield in kfcustomfields) - { - var localdigicertfieldinstance = new DigicertCustomFieldInstance(); - localdigicertfieldinstance.label = kfcustomfield.DigicertFieldName; - localdigicertfieldinstance.is_active = true; - localdigicertfieldinstance.kf_field_name = kfcustomfield.KeyfactorMetadataFieldName; - if (kfcustomfield.KeyfactorDataType == "String") - localdigicertfieldinstance.data_type = "text"; - else if (kfcustomfield.KeyfactorDataType == "Int") - localdigicertfieldinstance.data_type = "int"; - else - localdigicertfieldinstance.data_type = "anything"; - localdigicertfieldinstance.is_required = false; + localdigicertfieldinstance.label = dgcustomfield.label; + localdigicertfieldinstance.is_active = dgcustomfield.is_active; + localdigicertfieldinstance.data_type = dgcustomfield.data_type; + localdigicertfieldinstance.is_required = dgcustomfield.is_required; + + foreach (var kffieldeq in kfcustomfields) + if (dgcustomfield.label == kffieldeq.DigicertFieldName) + localdigicertfieldinstance.kf_field_name = kffieldeq.DigicertFieldName; - if (!fullcustomdgfieldlist.Any(p => p.label == localdigicertfieldinstance.label)) - { fullcustomdgfieldlist.Add(localdigicertfieldinstance); - newcustomfieldsfordg.Add(localdigicertfieldinstance); } - } - //Add fields that don't exist on DigiCert to Digicert - _logger.Trace("Adding following fields to DigiCert: {0}", JsonConvert.SerializeObject(newcustomfieldsfordg)); - foreach (var newdgfield in newcustomfieldsfordg) - { - var digicertapilocation = "https://www.digicert.com/services/v2/account/metadata"; - var digicertnewfieldsclient = new RestClient(); - var digicertnewfieldsrequest = new RestRequest(digicertapilocation); - digicertnewfieldsrequest.AddHeader("Accept", "application/json"); - digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikeytopperm); - var serializedsyncfield = JsonConvert.SerializeObject(newdgfield); - digicertnewfieldsrequest.AddParameter("application/json", serializedsyncfield, - ParameterType.RequestBody); - var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); - } + //This covers all of the new fields on Keyfactors side, including new ones - needs to have digicert ids for the new ones + foreach (var kfcustomfield in kfcustomfields) + { + var localdigicertfieldinstance = new DigicertCustomFieldInstance(); + localdigicertfieldinstance.label = kfcustomfield.DigicertFieldName; + localdigicertfieldinstance.is_active = true; + localdigicertfieldinstance.kf_field_name = kfcustomfield.KeyfactorMetadataFieldName; + if (kfcustomfield.KeyfactorDataType == "String") + localdigicertfieldinstance.data_type = "text"; + else if (kfcustomfield.KeyfactorDataType == "Int") + localdigicertfieldinstance.data_type = "int"; + else + localdigicertfieldinstance.data_type = "anything"; + localdigicertfieldinstance.is_required = false; + + if (!fullcustomdgfieldlist.Any(p => p.label == localdigicertfieldinstance.label)) + { + fullcustomdgfieldlist.Add(localdigicertfieldinstance); + newcustomfieldsfordg.Add(localdigicertfieldinstance); + } + } - // Grabbing the list again from digicert, populating ids for new ones - //Getting list of custom metadata fields on DigiCert - var updatedmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey, importdeactivated); - foreach (var subitem in updatedmetadatafieldlist) - foreach (var fulllistitem in fullcustomdgfieldlist) - if (subitem.label == fulllistitem.label) - fulllistitem.id = subitem.id; + //Add fields that don't exist on DigiCert to Digicert + _logger.Trace("Adding following fields to DigiCert: {0}", + JsonConvert.SerializeObject(newcustomfieldsfordg)); + foreach (var newdgfield in newcustomfieldsfordg) + { + var digicertapilocation = "https://www.digicert.com/services/v2/account/metadata"; + var digicertnewfieldsclient = new RestClient(); + var digicertnewfieldsrequest = new RestRequest(digicertapilocation); + digicertnewfieldsrequest.AddHeader("Accept", "application/json"); + digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikeytopperm); + var serializedsyncfield = JsonConvert.SerializeObject(newdgfield); + digicertnewfieldsrequest.AddParameter("application/json", serializedsyncfield, + ParameterType.RequestBody); + var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); + } - var totalcertsprocessed = 0; - var numcertsdatauploaded = 0; - // Pushing the data to DigiCert - var certlist2 = JsonConvert.DeserializeObject(rawresponse, - new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - foreach (var cert in certlist2) - { - Dictionary kfstoredmetadata = cert["Metadata"].ToObject>(); + // Grabbing the list again from digicert, populating ids for new ones + //Getting list of custom metadata fields on DigiCert + var updatedmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey, importdeactivated); + foreach (var subitem in updatedmetadatafieldlist) + foreach (var fulllistitem in fullcustomdgfieldlist) + if (subitem.label == fulllistitem.label) + fulllistitem.id = subitem.id; - var certhascustomfields = false; - foreach (var checkfield in fullcustomdgfieldlist) - if (kfstoredmetadata.ContainsKey(checkfield.kf_field_name)) - certhascustomfields = true; - if (certhascustomfields) + // Pushing the data to DigiCert + var certlist2 = JsonConvert.DeserializeObject(rawresponse, + new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + foreach (var cert in certlist2) { - var kfserialnumber = cert["SerialNumber"].ToString(); + Dictionary kfstoredmetadata = + cert["Metadata"].ToObject>(); - var digicertnewlookupurl = "https://www.digicert.com/services/v2/order/certificate" + - "?filters[serial_number]=" + kfserialnumber; + var certhascustomfields = false; + foreach (var checkfield in fullcustomdgfieldlist) + if (kfstoredmetadata.ContainsKey(checkfield.kf_field_name)) + certhascustomfields = true; - var newbodytemplate = new RootDigicertLookup(); - var newsearchcriterioninstance = new SearchCriterion(); - newbodytemplate.searchCriteriaList.Add(newsearchcriterioninstance); - var lookupnewrequest = new RestRequest(digicertnewlookupurl); - lookupnewrequest.AddHeader("Content-Type", "application/json"); - lookupnewrequest.AddHeader("X-DC-DEVKEY", digicertapikey); - var digicertnewlookupresponse = client.Execute(lookupnewrequest); - var newparseddigicertresponse = - JsonConvert.DeserializeObject(digicertnewlookupresponse.Content); + if (certhascustomfields) + { + var kfserialnumber = cert["SerialNumber"].ToString(); + var digicertnewlookupurl = "https://www.digicert.com/services/v2/order/certificate" + + "?filters[serial_number]=" + kfserialnumber; - if (newparseddigicertresponse["page"]["total"] != 0) - { - var newflatteneddigicertinstance = newparseddigicertresponse["orders"][0]; - var orderid = newflatteneddigicertinstance["id"].ToString(); + var newbodytemplate = new RootDigicertLookup(); + var newsearchcriterioninstance = new SearchCriterion(); + newbodytemplate.searchCriteriaList.Add(newsearchcriterioninstance); + var lookupnewrequest = new RestRequest(digicertnewlookupurl); + lookupnewrequest.AddHeader("Content-Type", "application/json"); + lookupnewrequest.AddHeader("X-DC-DEVKEY", digicertapikey); + var digicertnewlookupresponse = client.Execute(lookupnewrequest); + var newparseddigicertresponse = + JsonConvert.DeserializeObject(digicertnewlookupresponse.Content); - var digicertmetadataupdateapilocation = - "https://www.digicert.com/services/v2/order/certificate/" + orderid + "/custom-field"; - var digicertnewfieldsclient = new RestClient(); - var digicertnewfieldsrequest = new RestRequest(digicertmetadataupdateapilocation); - digicertnewfieldsrequest.AddHeader("Accept", "application/json"); - digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikey); - foreach (var newfield in fullcustomdgfieldlist) + if (newparseddigicertresponse["page"]["total"] != 0) { - var keyfactorfieldname = ""; - var datauploaded = false; - //Lookup the keyfactor name for digicert fields - foreach (var sublookup in kfcustomfields) - if (sublookup.DigicertFieldName == newfield.label) - { - var metadatapayload = new Dictionary(); - metadatapayload["metadata_id"] = newfield.id.ToString(); - if (kfstoredmetadata.ContainsKey(sublookup.KeyfactorMetadataFieldName)) + var newflatteneddigicertinstance = newparseddigicertresponse["orders"][0]; + var orderid = newflatteneddigicertinstance["id"].ToString(); + + var digicertmetadataupdateapilocation = + "https://www.digicert.com/services/v2/order/certificate/" + orderid + "/custom-field"; + var digicertnewfieldsclient = new RestClient(); + var digicertnewfieldsrequest = new RestRequest(digicertmetadataupdateapilocation); + digicertnewfieldsrequest.AddHeader("Accept", "application/json"); + digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikey); + + foreach (var newfield in fullcustomdgfieldlist) + { + var keyfactorfieldname = ""; + var datauploaded = false; + //Lookup the keyfactor name for digicert fields + foreach (var sublookup in kfcustomfields) + if (sublookup.DigicertFieldName == newfield.label) { - metadatapayload["value"] = - kfstoredmetadata[sublookup.KeyfactorMetadataFieldName]; - var newserializedsyncfield = JsonConvert.SerializeObject(metadatapayload); - digicertnewfieldsrequest.AddParameter("application/json", - newserializedsyncfield, ParameterType.RequestBody); - var digicertresponsenewfields = - digicertnewfieldsclient.Post(digicertnewfieldsrequest); - datauploaded = true; + var metadatapayload = new Dictionary(); + metadatapayload["metadata_id"] = newfield.id.ToString(); + if (kfstoredmetadata.ContainsKey(sublookup.KeyfactorMetadataFieldName)) + { + metadatapayload["value"] = + kfstoredmetadata[sublookup.KeyfactorMetadataFieldName]; + var newserializedsyncfield = JsonConvert.SerializeObject(metadatapayload); + digicertnewfieldsrequest.AddParameter("application/json", + newserializedsyncfield, ParameterType.RequestBody); + var digicertresponsenewfields = + digicertnewfieldsclient.Post(digicertnewfieldsrequest); + datauploaded = true; + } } - } - } + } - numcertsdatauploaded += 1; + numcertsdatauploaded += 1; + } } + + totalcertsprocessed += 1; } - totalcertsprocessed += 1; - } - Console.WriteLine( - $"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); - Console.WriteLine($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); - _logger.Debug( - $"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); - _logger.Debug($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); + // Update the count of items downloaded so far + certsdownloaded += batchsize; + + // Check if all items have been downloaded + if (certlist.Count == 0) + { + Console.WriteLine( + $"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); + Console.WriteLine($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); + _logger.Debug( + $"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); + _logger.Debug($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); ; + + break; + } + } } // Syncing Data from DigiCert TO Keyfactor if (config_mode == "dctokf") { - Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - _logger.Debug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - //Each cert that is DigiCert in origin in Keyfactor is looked up on DigiCert via serial number, - //and the metadata contents from those fields are stored. - var digicertlookupclient = new RestClient(); - var digicertcertificates = new List(); - foreach (var certinstance in certlist) + // Initialize variable to keep track of items downloaded so far + int certsdownloaded = 0; + var certcounttracker = 0; + for (int batchnum = 0; batchnum < numberOfBatches; batchnum++) { - var digicertlookupurl = "https://www.digicert.com/services/v2/order/certificate/"; - - var bodytemplate = new RootDigicertLookup(); - var searchcriterioninstance = new SearchCriterion(); - bodytemplate.searchCriteriaList.Add(searchcriterioninstance); - - digicertlookupurl = digicertlookupurl + certinstance.SerialNumber; - var lookuprequest = new RestRequest(digicertlookupurl); - lookuprequest.AddHeader("Content-Type", "application/json"); - lookuprequest.AddHeader("X-DC-DEVKEY", digicertapikey); - var digicertlookupresponse = client.Execute(lookuprequest); - var certcontent = JsonConvert.DeserializeObject(digicertlookupresponse.Content, - new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - - if (certcontent["certificate"] != null) + // Check if reaching the arbitrary limit + if (certsdownloaded + batchsize > returnlimitint) { - digicertcertificates.Add(certcontent); - _logger.Trace("Pulled and storing following cert from digicert: {0}", - digicertlookupresponse.Content); + Console.WriteLine($"Stopped downloading at the configured limit of {returnlimitint} items."); + _logger.Debug($"Stopped downloading at the configured limit of {returnlimitint} items."); + break; } - Console.WriteLine("Pulled DigiCert matching DigiCert cert data."); - _logger.Debug("Pulled DigiCert matching DigiCert cert data."); + // Download the items in this batch + Console.WriteLine($"Downloading batch {batchnum + 1}..."); - var certcounttracker = 0; - foreach (var digicertcertinstance in digicertcertificates) - { - var finalsyncclient = new RestClient(); - finalsyncclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); - var finalsyncurl = keyfactorapilocation + "Certificates/Metadata"; - //Find matching certificate via Keyfactor ID - var test = digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper(); - var query = from kfcertlocal in certlist - where kfcertlocal.SerialNumber == - digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper() - select kfcertlocal; - var certificateid = query.FirstOrDefault().Id; - } + var digicertlookup = keyfactorapilocation + "Certificates?pq.queryString=IssuerDN%20-contains%20%22" + + digicertIssuerQueryterm + "%22&pq.returnLimit=" + batchsize.ToString() + + "&includeMetadata=true" + "&pq.pageReturned=" + batchnum.ToString(); + var request = new RestRequest(digicertlookup); + request.AddHeader("Accept", "application/json"); + request.AddHeader("x-keyfactor-api-version", "1"); + request.AddHeader("x-keyfactor-requested-with", "APIClient"); + var response = client.Execute(request); + var rawresponse = response.Content; + var certlist = JsonConvert.DeserializeObject>(rawresponse, + new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + Console.WriteLine("Got DigiCert issued certs from keyfactor"); + _logger.Debug("Got DigiCert issued certs from keyfactor"); + + //Each cert that is DigiCert in origin in Keyfactor is looked up on DigiCert via serial number, + //and the metadata contents from those fields are stored. + var digicertlookupclient = new RestClient(); + var digicertcertificates = new List(); + foreach (var certinstance in certlist) + { + var digicertlookupurl = "https://www.digicert.com/services/v2/order/certificate/"; + var bodytemplate = new RootDigicertLookup(); + var searchcriterioninstance = new SearchCriterion(); + bodytemplate.searchCriteriaList.Add(searchcriterioninstance); - if (digicertcertinstance["custom_fields"] != null) - // Getting custom metadata field values - foreach (var metadatafieldinstance in digicertcertinstance["custom_fields"]) - if (importallcustomdigicertfields) + digicertlookupurl = digicertlookupurl + certinstance.SerialNumber; + var lookuprequest = new RestRequest(digicertlookupurl); + lookuprequest.AddHeader("Content-Type", "application/json"); + lookuprequest.AddHeader("X-DC-DEVKEY", digicertapikey); + var digicertlookupresponse = client.Execute(lookuprequest); + var certcontent = JsonConvert.DeserializeObject(digicertlookupresponse.Content, + new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + + if (certcontent["certificate"] != null) { - // Using autoimport and thus using autorename - var metadatanamefield = ReplaceAllWhiteSpaces(metadatafieldinstance["label"].ToString(), - replacementcharacter); - payloadforkf.Metadata[metadatanamefield] = metadatafieldinstance["value"]; + digicertcertificates.Add(certcontent); + _logger.Trace("Pulled and storing following cert from digicert: {0}", + digicertlookupresponse.Content); } - } - - } - - var totalcertsprocessed = 0; - var numcertsdatauploaded = 0; - - // Pushing the data to DigiCert - var certlist2 = JsonConvert.DeserializeObject(rawresponse, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - foreach (var cert in certlist2) - { - - Dictionary kfstoredmetadata = cert["Metadata"].ToObject>(); - - bool certhascustomfields = false; - foreach (var checkfield in fullcustomdgfieldlist) - { - if (kfstoredmetadata.ContainsKey(checkfield.kf_field_name)) + else { - certhascustomfields = true; + _logger.Trace("Failed to retrieve cert {0} from Digicert", digicertlookupresponse.Content); } } - if (certhascustomfields){ - var kfserialnumber = cert["SerialNumber"].ToString(); + Console.WriteLine("Pulled DigiCert matching DigiCert cert data."); + _logger.Debug("Pulled DigiCert matching DigiCert cert data."); - var digicertnewlookupurl = "https://www.digicert.com/services/v2/order/certificate" + "?filters[serial_number]=" + kfserialnumber; + foreach (var digicertcertinstance in digicertcertificates) + { + var finalsyncclient = new RestClient(); + finalsyncclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); + var finalsyncurl = keyfactorapilocation + "Certificates/Metadata"; + //Find matching certificate via Keyfactor ID + var test = digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper(); + var query = from kfcertlocal in certlist + where kfcertlocal.SerialNumber == + digicertcertinstance["certificate"]["serial_number"].ToString().ToUpper() + select kfcertlocal; + var certificateid = query.FirstOrDefault().Id; + + + var payloadforkf = new KeyfactorMetadataQuery(); + payloadforkf.Id = certificateid; + + if (digicertcertinstance["custom_fields"] != null) + // Getting custom metadata field values + foreach (var metadatafieldinstance in digicertcertinstance["custom_fields"]) + if (importallcustomdigicertfields) + { + // Using autoimport and thus using autorename + var metadatanamefield = ReplaceAllBannedCharacters(metadatafieldinstance["label"].ToString(), + allBannedChars); + payloadforkf.Metadata[metadatanamefield] = metadatafieldinstance["value"]; + } + else + { + //Using custom names + var metadatanamequery = from customfieldinstance in kfcustomfields + where customfieldinstance.DigicertFieldName == + metadatafieldinstance["label"] + select customfieldinstance; + if (metadatanamequery.FirstOrDefault() != null) + payloadforkf.Metadata[metadatanamequery.FirstOrDefault().DigicertFieldName] = + metadatafieldinstance["value"]; + } - var newbodytemplate = new RootDigicertLookup(); - var newsearchcriterioninstance = new SearchCriterion(); - newbodytemplate.searchCriteriaList.Add(newsearchcriterioninstance); - var lookupnewrequest = new RestRequest(digicertnewlookupurl); - lookupnewrequest.AddHeader("Content-Type", "application/json"); - lookupnewrequest.AddHeader("X-DC-DEVKEY", digicertapikey); - var digicertnewlookupresponse = client.Execute(lookupnewrequest); - var newparseddigicertresponse = JsonConvert.DeserializeObject(digicertnewlookupresponse.Content); + var flattenedcert = Flatten(digicertcertinstance); + //Getting manually selected metadata field values (not custom in DigiCert) + foreach (var manualinstance in kfmanualfields) + if (flattenedcert[manualinstance.DigicertFieldName] != null) + payloadforkf.Metadata[manualinstance.KeyfactorMetadataFieldName] = + flattenedcert[manualinstance.DigicertFieldName].ToString(); + //Sending the payload off to Keyfactor for the update + var finalsyncreq = new RestRequest(finalsyncurl); + finalsyncreq.AddHeader("Content-Type", "application/json"); + finalsyncreq.AddHeader("x-keyfactor-api-version", "1"); + finalsyncreq.AddHeader("x-keyfactor-requested-with", "APIClient"); + var serializedsyncfield = JsonConvert.SerializeObject(payloadforkf); + _logger.Trace("Sending Metadata update to KF for cert ID {0}, metadata update: {1}", + payloadforkf.Id.ToString(), serializedsyncfield); + + finalsyncreq.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); + finalsyncclient.Put(finalsyncreq); + ++certcounttracker; + } + - if (newparseddigicertresponse["page"]["total"] != 0) - { - //Using custom names - var metadatanamequery = from customfieldinstance in kfcustomfields - where customfieldinstance.DigicertFieldName == - metadatafieldinstance["label"] - select customfieldinstance; - if (metadatanamequery.FirstOrDefault() != null) - payloadforkf.Metadata[metadatanamequery.FirstOrDefault().DigicertFieldName] = - metadatafieldinstance["value"]; - } - } - totalcertsprocessed += 1; - } - Console.WriteLine($"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); - Console.WriteLine($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); - logger.LogDebug($"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); - logger.LogDebug($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); + // Update the count of items downloaded so far + certsdownloaded += batchsize; - } + // Check if all items have been downloaded + if (certlist.Count == 0) + { + Console.WriteLine( + $"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); + _logger.Debug( + $"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); - var flattenedcert = Flatten(digicertcertinstance); - //Getting manually selected metadata field values (not custom in DigiCert) - foreach (var manualinstance in kfmanualfields) - if (flattenedcert[manualinstance.DigicertFieldName] != null) - payloadforkf.Metadata[manualinstance.KeyfactorMetadataFieldName] = - flattenedcert[manualinstance.DigicertFieldName].ToString(); - //Sending the payload off to Keyfactor for the update - var finalsyncreq = new RestRequest(finalsyncurl); - finalsyncreq.AddHeader("Content-Type", "application/json"); - finalsyncreq.AddHeader("x-keyfactor-api-version", "1"); - finalsyncreq.AddHeader("x-keyfactor-requested-with", "APIClient"); - var serializedsyncfield = JsonConvert.SerializeObject(payloadforkf); - _logger.Trace("Sending Metadata update to KF for cert ID {0}, metadata update: {1}", - payloadforkf.Id.ToString(), serializedsyncfield); - - finalsyncreq.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); - finalsyncclient.Put(finalsyncreq); - ++certcounttracker; + break; + } } - - Console.WriteLine( - $"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); - _logger.Debug( - $"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); } + + Environment.Exit(0); } } \ No newline at end of file diff --git a/digicert-metadata-sync/Models/CharDBItem.cs b/digicert-metadata-sync/Models/CharDBItem.cs new file mode 100644 index 0000000..63e12c9 --- /dev/null +++ b/digicert-metadata-sync/Models/CharDBItem.cs @@ -0,0 +1,26 @@ +// Copyright 2021 Keyfactor +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace DigicertMetadataSync; + +partial class DigicertSync +{ + + public class CharDBItem + { + public string character; + public string replacementcharacter; + } + +} \ No newline at end of file diff --git a/digicert-metadata-sync/Models/DigicertCustomField.cs b/digicert-metadata-sync/Models/DigicertCustomField.cs index 3e2def7..a622ce7 100644 --- a/digicert-metadata-sync/Models/DigicertCustomField.cs +++ b/digicert-metadata-sync/Models/DigicertCustomField.cs @@ -14,8 +14,9 @@ namespace DigicertMetadataSync; -internal partial class DigicertSync +partial class DigicertSync { + public class DigicertCustomFieldInstance { public int id { get; set; } = 999999999; @@ -29,6 +30,7 @@ public class DigicertCustomFieldInstance public class DigicertMetadataUpdateInstance { public int metadata_id { get; set; } = 999999999; - public string value { get; set; } = "false"; + public string value { get; set; } = "false"; } + } \ No newline at end of file diff --git a/digicert-metadata-sync/Models/DigicertLookup.cs b/digicert-metadata-sync/Models/DigicertLookup.cs index 365c43e..d3f01c4 100644 --- a/digicert-metadata-sync/Models/DigicertLookup.cs +++ b/digicert-metadata-sync/Models/DigicertLookup.cs @@ -14,19 +14,19 @@ namespace DigicertMetadataSync; -internal partial class DigicertSync +partial class DigicertSync { public class RootDigicertLookup { - public List searchCriteriaList = new(); + public List searchCriteriaList = new List(); public string accountId { get; set; } = "x"; } public class SearchCriterion { - public List value = new(); public string key { get; set; } = "serialNum"; public string operation { get; set; } = "EQUALS"; + public List value = new(); } } \ No newline at end of file diff --git a/digicert-metadata-sync/Models/InternalClasses.cs b/digicert-metadata-sync/Models/InternalClasses.cs index 1cf7082..e015475 100644 --- a/digicert-metadata-sync/Models/InternalClasses.cs +++ b/digicert-metadata-sync/Models/InternalClasses.cs @@ -14,7 +14,7 @@ namespace DigicertMetadataSync; -internal partial class DigicertSync +partial class DigicertSync { public class CustomDigicertMetadataInstance { @@ -34,6 +34,7 @@ public class ReadInMetadataField public string KeyfactorDataType { get; set; } = "string"; public string KeyfactorHint { get; set; } = "None."; public string KeyfactorAllowAPI { get; set; } = "True"; + public string FieldType { get; set; } = "manual/custom"; } public class KeyfactorMetadataInstanceSendoff diff --git a/digicert-metadata-sync/Models/KeyfactorCertInstance.cs b/digicert-metadata-sync/Models/KeyfactorCertInstance.cs index 4188851..11e3670 100644 --- a/digicert-metadata-sync/Models/KeyfactorCertInstance.cs +++ b/digicert-metadata-sync/Models/KeyfactorCertInstance.cs @@ -14,7 +14,7 @@ namespace DigicertMetadataSync; -internal partial class DigicertSync +partial class DigicertSync { public class KeyfactorCert { diff --git a/digicert-metadata-sync/replacechar.json b/digicert-metadata-sync/replacechar.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/digicert-metadata-sync/replacechar.json @@ -0,0 +1 @@ +[] \ No newline at end of file From 7b552b3955b9b73f0707adc988d339afe9b20b10 Mon Sep 17 00:00:00 2001 From: Mark Kachkaev <37276742+mkachk@users.noreply.github.com> Date: Mon, 16 Oct 2023 09:18:13 -0400 Subject: [PATCH 09/12] Update readme_source.md Added information on the new replacechar.json file. --- readme_source.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/readme_source.md b/readme_source.md index 662d007..d6342a3 100644 --- a/readme_source.md +++ b/readme_source.md @@ -34,11 +34,16 @@ This should include the Keyfactor API endpoint, of the format https://domain.com This should include the common prefix all DigiCert certs have in your Keyfactor instance. For example, "DigiCert" - ImportAllCustomDigicertFields This setting enables the tool to import all of the custom metadata fields included in DigiCert and sync all of their data. -- ReplaceDigicertWhiteSpaceCharacterInName -In case the ImportAllCustomDigicertFields setting is used, this is necessary to for metadata field label conversion. DigiCert supports spaces in labels and Keyfactor does not, so this replaces the spaces in the name with your character sequence of choice. +During the first run, the tool will scan the custom fields it will be importing for characters that are not supported in Keyfactor Metadata field names. +Each unsupported character will be shown in a file named "replacechar.json" and its replacement can be selected. If the values in the file are not populated, the tool will not run a second time. - ImportDataForDeactivatedDigiCertFields If this is enabled, custom metadata fields that were deactivated in DigiCert will also be synced, and the data stored in these fields in certificates will be too. +### replacechar.json settings +This file is populated during the first run of the tool if the ImportAllCustomDigicertFields setting is toggled. +The only text that needs replacing is shown as "null", and can be filled with any alphanumeric string. The "_" and "-" characters are also supported. + + ### manualfields.json settings This file is used to specify which metadata fields should be synced up. From 78a597a00319bc6d3a25d930aae7bb23e1b00d73 Mon Sep 17 00:00:00 2001 From: Keyfactor Date: Mon, 16 Oct 2023 13:18:43 +0000 Subject: [PATCH 10/12] Update generated README --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3ccc863..b599c9f 100644 --- a/README.md +++ b/README.md @@ -50,11 +50,16 @@ This should include the Keyfactor API endpoint, of the format https://domain.com This should include the common prefix all DigiCert certs have in your Keyfactor instance. For example, "DigiCert" - ImportAllCustomDigicertFields This setting enables the tool to import all of the custom metadata fields included in DigiCert and sync all of their data. -- ReplaceDigicertWhiteSpaceCharacterInName -In case the ImportAllCustomDigicertFields setting is used, this is necessary to for metadata field label conversion. DigiCert supports spaces in labels and Keyfactor does not, so this replaces the spaces in the name with your character sequence of choice. +During the first run, the tool will scan the custom fields it will be importing for characters that are not supported in Keyfactor Metadata field names. +Each unsupported character will be shown in a file named "replacechar.json" and its replacement can be selected. If the values in the file are not populated, the tool will not run a second time. - ImportDataForDeactivatedDigiCertFields If this is enabled, custom metadata fields that were deactivated in DigiCert will also be synced, and the data stored in these fields in certificates will be too. +### replacechar.json settings +This file is populated during the first run of the tool if the ImportAllCustomDigicertFields setting is toggled. +The only text that needs replacing is shown as "null", and can be filled with any alphanumeric string. The "_" and "-" characters are also supported. + + ### manualfields.json settings This file is used to specify which metadata fields should be synced up. From 853ba2e72d1a5b71bd2c9ef49511ced894d2c067 Mon Sep 17 00:00:00 2001 From: Mark Kachkaev <37276742+mkachk@users.noreply.github.com> Date: Mon, 16 Oct 2023 17:47:34 -0400 Subject: [PATCH 11/12] Fixed MetadataSync.cs compilation issue --- digicert-metadata-sync/MetadataSync.cs | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/digicert-metadata-sync/MetadataSync.cs b/digicert-metadata-sync/MetadataSync.cs index f0eb765..ce31580 100644 --- a/digicert-metadata-sync/MetadataSync.cs +++ b/digicert-metadata-sync/MetadataSync.cs @@ -1,11 +1,10 @@ -// Copyright 2021 Keyfactor +// Copyright 2021 Keyfactor // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. // You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 // Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions // and limitations under the License. - using System.Collections.Generic; using System.IO; using System.Reflection.Metadata.Ecma335; @@ -292,12 +291,10 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. if (dgcustomfield.label == kffieldeq.DigicertFieldName) localdigicertfieldinstance.kf_field_name = kffieldeq.DigicertFieldName; - numcertsdatauploaded += 1; - } + fullcustomdgfieldlist.Add(localdigicertfieldinstance); } - //This covers all of the new fields on Keyfactors side, including new ones - needs to have digicert ids for the new ones foreach (var kfcustomfield in kfcustomfields) { @@ -379,7 +376,6 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. if (newparseddigicertresponse["page"]["total"] != 0) { - var newflatteneddigicertinstance = newparseddigicertresponse["orders"][0]; var orderid = newflatteneddigicertinstance["id"].ToString(); @@ -415,7 +411,6 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. } numcertsdatauploaded += 1; - } } @@ -586,12 +581,8 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. break; } } - - Console.WriteLine( - $"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); - _logger.Debug( - $"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); } + Environment.Exit(0); } -} \ No newline at end of file +} From 934a46f152730d2a568d448544e096acb0d05a2a Mon Sep 17 00:00:00 2001 From: Mark Kachkaev <37276742+mkachk@users.noreply.github.com> Date: Mon, 16 Oct 2023 17:49:03 -0400 Subject: [PATCH 12/12] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 796d4f1..6804b4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +Version 2.1.0 + + Added a system that gathers all non-Keyfactor friendly characters and allows the user to configure an alternative. + Added pagination based batch processing, memory consumption has been drastically reduced. + Version 2.0.3 Added a setting to enable or disable syncing deactivated custom fields from DigiCert.