From aeec45aa481feb711e058f0f0e9e0d01d1e5e3e7 Mon Sep 17 00:00:00 2001 From: Mark Kachkaev Date: Tue, 28 Mar 2023 15:55:38 -0400 Subject: [PATCH 1/6] Version 2.0 - added sync from KF to DC. --- CHANGELOG.md | 8 +- README.md | 13 +- .../AddFieldsToKeyfactor.cs | 8 +- digicert-metadata-sync/App.config | 5 +- .../GrabCustomFieldsFromDigiCert.cs | 44 ++ digicert-metadata-sync/Helpers.cs | 8 + digicert-metadata-sync/MetadataSync.cs | 387 +++++++++++++----- .../Models/DigicertCustomField.cs | 36 ++ .../Models/KeyfactorCertInstance.cs | 1 + readme_source.md | 16 +- 10 files changed, 411 insertions(+), 115 deletions(-) create mode 100644 digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs create mode 100644 digicert-metadata-sync/Models/DigicertCustomField.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 685852d..8247963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +Version 2.0 + +- Added ability to sync custom fields from Keyfactor to DigiCert. +- Tool now requires command line argument to specify sync direction: "dctokf" for DigiCert to Keyfactor and "kftodc" for Keyfactor to DigiCert. +- New DigiCert API Key with restrictions set to "None" in DigiCert config required to perform sync from Keyfactor to Digicert. + Version 1.0 -Initial Release +- Initial Release diff --git a/README.md b/README.md index 9b40160..ef850f0 100644 --- a/README.md +++ b/README.md @@ -21,18 +21,27 @@ ___ ## Overview -This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. +This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. The latest update allows for syncronization of custom field contents from Keyfactor to DigiCert. New fields are created in Keyfactor and DigiCert to accomodate for this. ## Installation and Usage The tool comes as a Windows executable. The tool performs synchronization each time its run. For the tool to run automatically, it needs to be added as a scheduled process using Windows. The advised interval for running it is once per week. The files App.config and manualfields.json need to be present in the same directory as the tool for it to run correctly. The specific location from which the tool is ran does not matter, but it needs to have access to both the Keyfactor API endpoint as well as Digicert, and appropriate permissions for access to the configuration files. An explanation for the settings found in these files is given below. +## Command Line Arguments +One of these two arguments needs to be used for the tool to run. +- "kftodc" +Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. +- "dctokf" +Syncronizes the contents of both custom and non-custom fields from DigiCert to Keyfactor. The fields are listed in manualfields.json, and are created if necessary. + ## Settings The settings currently present in these files are shown as an example and need to be configured for your specific situation. ### app.config settings - DigicertAPIKey -Standard DigiCert API access key +Standard DigiCert API access key. +- DigicertAPIKeyTopPerm +DigiCert API access key with restrictions set to "None" - required for sync from Keyfactor to DigiCert. - KeyfactorDomainAndUser Same credential as used when logging into Keyfactor Command. A different set of credentials can be used provided they have adequate access permissions. - KeyfactorPassword diff --git a/digicert-metadata-sync/AddFieldsToKeyfactor.cs b/digicert-metadata-sync/AddFieldsToKeyfactor.cs index 33289d3..363da96 100644 --- a/digicert-metadata-sync/AddFieldsToKeyfactor.cs +++ b/digicert-metadata-sync/AddFieldsToKeyfactor.cs @@ -22,7 +22,7 @@ namespace DigicertMetadataSync; // It will only add new fields. partial class DigicertSync { - public static int AddFieldsToKeyfactor(List inputlist, + public static Tuple> AddFieldsToKeyfactor(List inputlist, List existingmetadatalist, bool noexistingfields, string keyfactorusername, string keyfactorpassword, string keyfactorapilocation) { @@ -30,6 +30,7 @@ public static int AddFieldsToKeyfactor(List in var addfieldsclient = new RestClient(); addfieldsclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); int totalnumberadded = 0; + List newfields = new List(); foreach (var metadatainstance in inputlist) { if (noexistingfields == false) @@ -37,6 +38,7 @@ public static int AddFieldsToKeyfactor(List in var fieldquery = from existingmetadatainstance in existingmetadatalist where existingmetadatainstance.Name == metadatainstance.Name select existingmetadatainstance; + // If field does not exist in Keyfactor, add it. if (!fieldquery.Any()) { var addfieldrequest = new RestRequest(addfieldstokeyfactorurl); @@ -50,6 +52,7 @@ public static int AddFieldsToKeyfactor(List in try { metadataresponse = addfieldsclient.Post(addfieldrequest); + newfields.Add(metadatainstance.Name); ++totalnumberadded; } catch (HttpRequestException e) @@ -90,7 +93,8 @@ public static int AddFieldsToKeyfactor(List in } } } + Tuple> returnvals = new Tuple>(totalnumberadded, newfields); - return totalnumberadded; + return returnvals; } } \ No newline at end of file diff --git a/digicert-metadata-sync/App.config b/digicert-metadata-sync/App.config index 40e5a24..3d97fda 100644 --- a/digicert-metadata-sync/App.config +++ b/digicert-metadata-sync/App.config @@ -2,10 +2,11 @@ + - - + + diff --git a/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs b/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs new file mode 100644 index 0000000..834097e --- /dev/null +++ b/digicert-metadata-sync/GrabCustomFieldsFromDigiCert.cs @@ -0,0 +1,44 @@ +// Copyright 2021 Keyfactor +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Keyfactor.Logging; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using RestSharp; +using RestSharp.Authenticators; + +namespace DigicertMetadataSync; + +// This fuction adds the fields to keyfactor. +// It will only add new fields. +partial class DigicertSync +{ + public static List GrabCustomFieldsFromDigiCert(string apikey) + { + ILogger logger = LogHandler.GetClassLogger(); + var digicertclient = new RestClient(); + var customfieldsretrieval = "https://www.digicert.com/services/v2/account/metadata"; + var digicertrequest = new RestRequest(customfieldsretrieval); + digicertrequest.AddHeader("Accept", "application/json"); + digicertrequest.AddHeader("X-DC-DEVKEY", apikey); + var digicertresponse = digicertclient.Execute(digicertrequest); + var trimmeddigicertresponse = digicertresponse.Content.Remove(0, 12); + int lengthofresponse = trimmeddigicertresponse.Length; + trimmeddigicertresponse = trimmeddigicertresponse.Remove(lengthofresponse - 1, 1); + var fieldlist = JsonConvert.DeserializeObject>(trimmeddigicertresponse); + Console.WriteLine("Obtained custom fields from DigiCert."); + logger.LogDebug("Obtained custom fields from DigiCert."); + return fieldlist; + } +} \ No newline at end of file diff --git a/digicert-metadata-sync/Helpers.cs b/digicert-metadata-sync/Helpers.cs index 2c5eaee..b44c192 100644 --- a/digicert-metadata-sync/Helpers.cs +++ b/digicert-metadata-sync/Helpers.cs @@ -70,6 +70,14 @@ public static string ReplaceAllWhiteSpaces(string str, string replacement) return Regex.Replace(str, @"\s+", "_-_"); } + public static bool CheckMode(string mode) + { + if ((mode == "kftodc") || (mode == "dctokf")){ + return true; + } + return false; + } + private static List convertlisttokf(List inputlist, string replacementcharacter) { diff --git a/digicert-metadata-sync/MetadataSync.cs b/digicert-metadata-sync/MetadataSync.cs index 4b0baa7..1e36077 100644 --- a/digicert-metadata-sync/MetadataSync.cs +++ b/digicert-metadata-sync/MetadataSync.cs @@ -25,6 +25,9 @@ using Keyfactor; using Microsoft.Extensions.Logging; using Keyfactor.Logging; +using static DigicertMetadataSync.DigicertSync; +using System.Collections.Immutable; +using System.Runtime.CompilerServices; namespace DigicertMetadataSync { @@ -36,19 +39,27 @@ public static void Main(string[] args) ILogger logger = LogHandler.GetClassLogger(); logger.LogDebug("Start sync"); var digicertapikey = System.Configuration.ConfigurationManager.AppSettings.Get("DigicertAPIKey"); + var digicertapikeytopperm = System.Configuration.ConfigurationManager.AppSettings.Get("DigicertAPIKeyTopPerm"); var keyfactorusername = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorDomainAndUser"); var keyfactorpassword = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorPassword"); var replacementcharacter = System.Configuration.ConfigurationManager.AppSettings.Get("ReplaceDigicertWhiteSpaceCharacterInName"); var importallcustomdigicertfields = Convert.ToBoolean(System.Configuration.ConfigurationManager.AppSettings.Get("ImportAllCustomDigicertFields")); - //Get list of all digicert certs from keyfactor based on query that contains digicert as issuer. + var config_mode = args[0]; + if (CheckMode(config_mode) == false) + { + logger.LogDebug("Inappropriate configuration mode. Check your command line arguments."); + throw new Exception("Inappropriate configuration mode. Check your command line arguments."); + } + + //Get list of all DigiCert certs from Keyfactor based on query that contains DigiCert as issuer. var returnlimit = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorCertSearchReturnLimit").ToString(); var keyfactorapilocation = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorAPIEndpoint").ToString(); var digicertIssuerQueryterm = System.Configuration.ConfigurationManager.AppSettings.Get("KeyfactorDigicertIssuedCertQueryTerm").ToString(); logger.LogDebug($"Loaded config. Processing with a Keyfactor Query Return Limit of {returnlimit.ToString()}"); var digicertlookup = keyfactorapilocation + "Certificates?pq.queryString=IssuerDN%20-contains%20%22" - + digicertIssuerQueryterm + "%22&pq.returnLimit=" + returnlimit; + + digicertIssuerQueryterm + "%22&pq.returnLimit=" + returnlimit + "&includeMetadata=true"; var client = new RestClient(); client.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); var request = new RestRequest(digicertlookup); @@ -58,8 +69,8 @@ public static void Main(string[] args) var response = client.Execute(request); var rawresponse = response.Content; var certlist = JsonConvert.DeserializeObject>(rawresponse, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - Console.WriteLine("Got digicert issued certs from keyfactor"); - logger.LogDebug("Got digicert issued certs from keyfactor"); + Console.WriteLine("Got DigiCert issued certs from keyfactor"); + logger.LogDebug("Got DigiCert issued certs from keyfactor"); //Getting list of custom metadata fields from Keyfactor var getmetadalistkf = keyfactorapilocation + "MetadataFields"; @@ -75,28 +86,17 @@ public static void Main(string[] args) Console.WriteLine("Got list of custom fields from Keyfactor."); logger.LogDebug("Got list of custom fields from Keyfactor."); - //Getting list of custom metadata fields on Digicert - var digicertclient = new RestClient(); - var customfieldsretrieval = "https://www.digicert.com/services/v2/account/metadata"; - var digicertrequest = new RestRequest(customfieldsretrieval); - digicertrequest.AddHeader("Accept", "application/json"); - digicertrequest.AddHeader("X-DC-DEVKEY", digicertapikey); - var digicertresponse = client.Execute(digicertrequest); - var trimmeddigicertresponse = digicertresponse.Content.Remove(0, 12); - int lengthofresponse = trimmeddigicertresponse.Length; - trimmeddigicertresponse = trimmeddigicertresponse.Remove(lengthofresponse - 1, 1); - var customdigicertmetadatafieldlist = JsonConvert.DeserializeObject>(trimmeddigicertresponse); - Console.WriteLine("Obtained custom fields from digicert."); - logger.LogDebug("Obtained custom fields from digicert."); - - //Convert digicert custom fields to keyfactor appropriate ones + //Getting list of custom metadata fields on DigiCert + var customdigicertmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey); + + //Convert DigiCert custom fields to Keyfactor appropriate ones //This depends on whether the setting to import all fields was enabled or not var config = new ConfigurationBuilder().SetBasePath(AppDomain.CurrentDomain.BaseDirectory).AddJsonFile("manualfields.json").Build(); var kfcustomfields = new List(); if (importallcustomdigicertfields == true) { - //This imports all the custom fields based on the list of metadata from Digicert and does autofill + //This imports all the custom fields based on the list of metadata from DigiCert and does autofill for (int i = 0; i < customdigicertmetadatafieldlist.Count; i++) { var localkffieldinstance = new ReadInMetadataField(); @@ -112,10 +112,10 @@ public static void Main(string[] args) if (customdigicertmetadatafieldlist[i].label != null) { /* - NOTICE: KEYFACTOR DOES NOT SUPPORT SPACES IN METADATA FIELD NAMES. + NOTICE: KEYFACTOR DOES NOT SUPPORT SPACES IN METADATA FIELD NAMES. WHITESPACE MUST BE REMOVED FROM THE NAME. CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. - */ + */ localkffieldinstance.DigicertFieldName = customdigicertmetadatafieldlist[i].label; localkffieldinstance.KeyfactorMetadataFieldName = ReplaceAllWhiteSpaces(customdigicertmetadatafieldlist[i].label, replacementcharacter); } @@ -136,7 +136,7 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. kfcustomfields = config.GetSection(customfieldslst).Get>(); } - //Adding metadata fields for the ID and the email of the requester from digicert. + //Adding metadata fields for the ID and the email of the requester from DigiCert. List kfmanualfields = new List(); var manualfieldslist = "ManualFields"; kfmanualfields = config.GetSection(manualfieldslist).Get>(); @@ -161,7 +161,7 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. noexistingfields = false; } Console.WriteLine("Pulled existing metadata fields from keyfactor."); - logger.LogDebug("Pulled existing metadata fields from keyfactor."); + logger.LogDebug("Pulled existing metadata fields from Keyfactor."); // Converting the read in fields into sendable lists var convertedmanualfields = convertlisttokf(kfmanualfields, replacementcharacter); var convertedcustomfields = convertlisttokf(kfcustomfields, replacementcharacter); @@ -169,119 +169,294 @@ CURRENTLY REPLACING WITH "_-_" AS STAND IN FOR SPACE CHARACTER. int totalfieldsadded = 0; //If all the fields are absent from Keyfactor, the fields are added. - totalfieldsadded += AddFieldsToKeyfactor(convertedmanualfields, existingmetadatalist, noexistingfields, keyfactorusername, keyfactorpassword, keyfactorapilocation); - totalfieldsadded += AddFieldsToKeyfactor(convertedcustomfields, existingmetadatalist, noexistingfields, keyfactorusername, keyfactorpassword, keyfactorapilocation); + var manualresult = AddFieldsToKeyfactor(convertedmanualfields, existingmetadatalist, noexistingfields, keyfactorusername, keyfactorpassword, keyfactorapilocation); + var customresult = AddFieldsToKeyfactor(convertedcustomfields, existingmetadatalist, noexistingfields, keyfactorusername, keyfactorpassword, keyfactorapilocation); + totalfieldsadded += manualresult.Item1; + totalfieldsadded += customresult.Item1; - Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - logger.LogDebug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); - //Each cert that is digicert in origin in keyfactor is looked up on Digicert via serial number, - //and the metadata contents from those fields are stored. - var digicertlookupclient = new RestClient(); - List digicertcertificates = new List(); - foreach (var certinstance in certlist) + var allnewfields = manualresult.Item2.Concat(customresult.Item2).ToList(); + // Syncing Data from Keyfactor TO DigiCert + // Sync from DigiCert to Keyfactor must run at least once prior to this - only runs with custom fields + if (config_mode == "kftodc") { - var digicertlookupurl = "https://www.digicert.com/services/v2/order/certificate/"; - - var bodytemplate = new RootDigicertLookup(); - var searchcriterioninstance = new SearchCriterion(); - bodytemplate.searchCriteriaList.Add(searchcriterioninstance); - - digicertlookupurl = digicertlookupurl + certinstance.SerialNumber; - var lookuprequest = new RestRequest(digicertlookupurl); - lookuprequest.AddHeader("Content-Type", "application/json"); - lookuprequest.AddHeader("X-DC-DEVKEY", digicertapikey); - var digicertlookupresponse = client.Execute(lookuprequest); - var parseddigicertresponse = JsonConvert.DeserializeObject(digicertlookupresponse.Content); - if (parseddigicertresponse.certificate != null) + Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + logger.LogDebug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + + List fullcustomdgfieldlist = new List(); + List newcustomfieldsfordg = new List(); + // Rebuild the list of metadata field names as they are on DigiCerts side. + + // This covers all of the custom fields on Digicerts side + foreach (var dgcustomfield in customdigicertmetadatafieldlist) { - var flatteneddigicertinstance = ClassConverter(parseddigicertresponse); - digicertcertificates.Add(parseddigicertresponse); + DigicertCustomFieldInstance localdigicertfieldinstance = new DigicertCustomFieldInstance(); + + localdigicertfieldinstance.label = dgcustomfield.label; + localdigicertfieldinstance.is_active = dgcustomfield.is_active; + localdigicertfieldinstance.data_type = dgcustomfield.data_type; + localdigicertfieldinstance.is_required = dgcustomfield.is_required; + + foreach (var kffieldeq in kfcustomfields) + { + if (dgcustomfield.label == kffieldeq.DigicertFieldName) + { + localdigicertfieldinstance.kf_field_name = kffieldeq.DigicertFieldName; + } + } + + fullcustomdgfieldlist.Add(localdigicertfieldinstance); } - } - Console.WriteLine("Pulled digicert matching digicert cert data."); - logger.LogDebug("Pulled digicert matching digicert cert data."); - //The metadata for each cert is served back to keyfactor and the fields are updated. + + //This covers all of the new fields on Keyfactors side, including new ones - needs to have digicert ids for the new ones + foreach (var kfcustomfield in kfcustomfields) + { + DigicertCustomFieldInstance localdigicertfieldinstance = new DigicertCustomFieldInstance(); + localdigicertfieldinstance.label = kfcustomfield.DigicertFieldName; + localdigicertfieldinstance.is_active = true; + localdigicertfieldinstance.kf_field_name = kfcustomfield.KeyfactorMetadataFieldName; + if (kfcustomfield.KeyfactorDataType == "String") + { + localdigicertfieldinstance.data_type = "text"; + } + else if (kfcustomfield.KeyfactorDataType == "Int") + { + localdigicertfieldinstance.data_type = "int"; + } + else + { + localdigicertfieldinstance.data_type = "anything"; + } + localdigicertfieldinstance.is_required = false; + + if (!fullcustomdgfieldlist.Any(p => p.label == localdigicertfieldinstance.label)) + { + fullcustomdgfieldlist.Add(localdigicertfieldinstance); + newcustomfieldsfordg.Add(localdigicertfieldinstance); + } + } - int certcounttracker = 0; - foreach (var digicertcertinstance in digicertcertificates) - { - var finalsyncclient = new RestClient(); - finalsyncclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); - var finalsyncurl = keyfactorapilocation+ "Certificates/Metadata"; - //Find matching certificate via Keyfactor ID - var query = from kfcertlocal in certlist - where kfcertlocal.SerialNumber == - digicertcertinstance.certificate.serial_number.ToUpper() - select kfcertlocal; - var certificateid = query.FirstOrDefault().Id; + //Add fields that don't exist on DigiCert to Digicert + foreach (var newdgfield in newcustomfieldsfordg) + { + var digicertapilocation = "https://www.digicert.com/services/v2/account/metadata"; + var digicertnewfieldsclient = new RestClient(); + var digicertnewfieldsrequest = new RestRequest(digicertapilocation); + digicertnewfieldsrequest.AddHeader("Accept", "application/json"); + digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikeytopperm); + var serializedsyncfield = JsonConvert.SerializeObject(newdgfield); + digicertnewfieldsrequest.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); + var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); + } - var payloadforkf = new KeyfactorMetadataQuery(); - payloadforkf.Id = certificateid; - if (digicertcertinstance.custom_fields != null) + // Grabbing the list again from digicert, populating ids for new ones + //Getting list of custom metadata fields on DigiCert + var updatedmetadatafieldlist = GrabCustomFieldsFromDigiCert(digicertapikey); + foreach (var subitem in updatedmetadatafieldlist) { - // Getting custom metadata field values - foreach (var metadatafieldinstance in digicertcertinstance.custom_fields) + foreach (var fulllistitem in fullcustomdgfieldlist) { - if (importallcustomdigicertfields == true) + if (subitem.label == fulllistitem.label) { - // Using autoimport and thus using autorename - payloadforkf.Metadata[ReplaceAllWhiteSpaces(metadatafieldinstance.label, replacementcharacter)] = metadatafieldinstance.value; + fulllistitem.id = subitem.id; } - else + } + + } + + var totalcertsprocessed = 0; + var numcertsdatauploaded = 0; + + // Pushing the data to DigiCert + var certlist2 = JsonConvert.DeserializeObject(rawresponse, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + foreach (var cert in certlist2) + { + + Dictionary kfstoredmetadata = cert["Metadata"].ToObject>(); + + bool certhascustomfields = false; + foreach (var checkfield in fullcustomdgfieldlist) + { + if (kfstoredmetadata.ContainsKey(checkfield.kf_field_name)) + { + certhascustomfields = true; + } + } + + if (certhascustomfields){ + var kfserialnumber = cert["SerialNumber"].ToString(); + + var digicertnewlookupurl = "https://www.digicert.com/services/v2/order/certificate" + "?filters[serial_number]=" + kfserialnumber; + + var newbodytemplate = new RootDigicertLookup(); + var newsearchcriterioninstance = new SearchCriterion(); + newbodytemplate.searchCriteriaList.Add(newsearchcriterioninstance); + var lookupnewrequest = new RestRequest(digicertnewlookupurl); + lookupnewrequest.AddHeader("Content-Type", "application/json"); + lookupnewrequest.AddHeader("X-DC-DEVKEY", digicertapikey); + var digicertnewlookupresponse = client.Execute(lookupnewrequest); + var newparseddigicertresponse = JsonConvert.DeserializeObject(digicertnewlookupresponse.Content); + + + if (newparseddigicertresponse["page"]["total"] != 0) { - //Using custom names - var metadatanamequery = from customfieldinstance in kfcustomfields - where customfieldinstance.DigicertFieldName == - metadatafieldinstance.label - select customfieldinstance; - if (metadatanamequery.FirstOrDefault() != null) + var newflatteneddigicertinstance = newparseddigicertresponse["orders"][0]; + var orderid = newflatteneddigicertinstance["id"].ToString(); + + var digicertmetadataupdateapilocation = "https://www.digicert.com/services/v2/order/certificate/" + orderid + "/custom-field"; + var digicertnewfieldsclient = new RestClient(); + var digicertnewfieldsrequest = new RestRequest(digicertmetadataupdateapilocation); + digicertnewfieldsrequest.AddHeader("Accept", "application/json"); + digicertnewfieldsrequest.AddHeader("X-DC-DEVKEY", digicertapikey); + + foreach (var newfield in fullcustomdgfieldlist) { - payloadforkf.Metadata[metadatanamequery.FirstOrDefault().DigicertFieldName] = metadatafieldinstance.value; + string keyfactorfieldname = ""; + bool datauploaded = false; + //Lookup the keyfactor name for digicert fields + foreach (var sublookup in kfcustomfields) + { + if (sublookup.DigicertFieldName == newfield.label) + { + Dictionary metadatapayload = new Dictionary(); + metadatapayload["metadata_id"] = newfield.id.ToString(); + //Update payload with data + metadatapayload["value"] = kfstoredmetadata[sublookup.DigicertFieldName]; + var newserializedsyncfield = JsonConvert.SerializeObject(metadatapayload); + digicertnewfieldsrequest.AddParameter("application/json", newserializedsyncfield, ParameterType.RequestBody); + var digicertresponsenewfields = digicertnewfieldsclient.Post(digicertnewfieldsrequest); + datauploaded = true; + } + } } - + numcertsdatauploaded += 1; } + } + totalcertsprocessed += 1; + } + Console.WriteLine($"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); + Console.WriteLine($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); + logger.LogDebug($"Metadata sync from Keyfactor to DigiCert complete. Number of certs processed: {totalcertsprocessed.ToString()}"); + logger.LogDebug($"Certs that had their metadata synced: {numcertsdatauploaded.ToString()}"); + } + + // Syncing Data from DigiCert TO Keyfactor + if (config_mode == "dctokf") + { + Console.WriteLine($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + logger.LogDebug($"Added custom fields to Keyfactor. Total fields added: {totalfieldsadded.ToString()}"); + //Each cert that is DigiCert in origin in Keyfactor is looked up on DigiCert via serial number, + //and the metadata contents from those fields are stored. + var digicertlookupclient = new RestClient(); + List digicertcertificates = new List(); + foreach (var certinstance in certlist) + { + var digicertlookupurl = "https://www.digicert.com/services/v2/order/certificate/"; + + var bodytemplate = new RootDigicertLookup(); + var searchcriterioninstance = new SearchCriterion(); + bodytemplate.searchCriteriaList.Add(searchcriterioninstance); + + digicertlookupurl = digicertlookupurl + certinstance.SerialNumber; + var lookuprequest = new RestRequest(digicertlookupurl); + lookuprequest.AddHeader("Content-Type", "application/json"); + lookuprequest.AddHeader("X-DC-DEVKEY", digicertapikey); + var digicertlookupresponse = client.Execute(lookuprequest); + var parseddigicertresponse = JsonConvert.DeserializeObject(digicertlookupresponse.Content); + if (parseddigicertresponse.certificate != null) + { + var flatteneddigicertinstance = ClassConverter(parseddigicertresponse); + digicertcertificates.Add(parseddigicertresponse); } } - var flatteneddigicertinstance = ClassConverter(digicertcertinstance); + Console.WriteLine("Pulled DigiCert matching DigiCert cert data."); + logger.LogDebug("Pulled DigiCert matching DigiCert cert data."); + - //Getting manually selected metadata field values (not custom in Digicert) - foreach (var manualinstance in kfmanualfields) + int certcounttracker = 0; + foreach (var digicertcertinstance in digicertcertificates) { - string[] access = manualinstance.DigicertFieldName.Split("."); + var finalsyncclient = new RestClient(); + finalsyncclient.Authenticator = new HttpBasicAuthenticator(keyfactorusername, keyfactorpassword); + var finalsyncurl = keyfactorapilocation + "Certificates/Metadata"; + //Find matching certificate via Keyfactor ID + var query = from kfcertlocal in certlist + where kfcertlocal.SerialNumber == + digicertcertinstance.certificate.serial_number.ToUpper() + select kfcertlocal; + var certificateid = query.FirstOrDefault().Id; - List keys = access.ToList(); - Dictionary recursionresult = recursiveopener(flatteneddigicertinstance, keys, keys.Count); - object value = new object(); - if (recursionresult != null) + + var payloadforkf = new KeyfactorMetadataQuery(); + payloadforkf.Id = certificateid; + + if (digicertcertinstance.custom_fields != null) { - value = recursionresult.First().Value; + // Getting custom metadata field values + foreach (var metadatafieldinstance in digicertcertinstance.custom_fields) + { + if (importallcustomdigicertfields == true) + { + // Using autoimport and thus using autorename + payloadforkf.Metadata[ReplaceAllWhiteSpaces(metadatafieldinstance.label, replacementcharacter)] = metadatafieldinstance.value; + } + else + { + //Using custom names + var metadatanamequery = from customfieldinstance in kfcustomfields + where customfieldinstance.DigicertFieldName == + metadatafieldinstance.label + select customfieldinstance; + if (metadatanamequery.FirstOrDefault() != null) + { + payloadforkf.Metadata[metadatanamequery.FirstOrDefault().DigicertFieldName] = metadatafieldinstance.value; + } + + } + + } } - else + + var flatteneddigicertinstance = ClassConverter(digicertcertinstance); + + //Getting manually selected metadata field values (not custom in DigiCert) + foreach (var manualinstance in kfmanualfields) { - value = ""; + string[] access = manualinstance.DigicertFieldName.Split("."); + + List keys = access.ToList(); + Dictionary recursionresult = recursiveopener(flatteneddigicertinstance, keys, keys.Count); + object value = new object(); + if (recursionresult != null) + { + value = recursionresult.First().Value; + } + else + { + value = ""; + } + payloadforkf.Metadata[manualinstance.KeyfactorMetadataFieldName] = value; } - payloadforkf.Metadata[manualinstance.KeyfactorMetadataFieldName] = value; + payloadforkf.Metadata["DigicertID"] = digicertcertinstance.id.ToString(); + //Sending the payload off to Keyfactor for the update + var finalsyncreq = new RestRequest(finalsyncurl); + finalsyncreq.AddHeader("Content-Type", "application/json"); + finalsyncreq.AddHeader("x-keyfactor-api-version", "1"); + finalsyncreq.AddHeader("x-keyfactor-requested-with", "APIClient"); + var serializedsyncfield = JsonConvert.SerializeObject(payloadforkf); + finalsyncreq.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); + finalsyncclient.Put(finalsyncreq); + ++certcounttracker; } - payloadforkf.Metadata["DigicertID"] = digicertcertinstance.id.ToString(); - //Sending the payload off to Keyfactor for the update - var finalsyncreq = new RestRequest(finalsyncurl); - finalsyncreq.AddHeader("Content-Type", "application/json"); - finalsyncreq.AddHeader("x-keyfactor-api-version", "1"); - finalsyncreq.AddHeader("x-keyfactor-requested-with", "APIClient"); - var serializedsyncfield = JsonConvert.SerializeObject(payloadforkf); - finalsyncreq.AddParameter("application/json", serializedsyncfield, ParameterType.RequestBody); - finalsyncclient.Put(finalsyncreq); - ++certcounttracker; - } - Console.WriteLine($"Metadata sync complete. Number of certs synced: {certcounttracker.ToString()}"); - logger.LogDebug($"Metadata sync complete. Number of certs synced: {certcounttracker.ToString()}"); + Console.WriteLine($"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); + logger.LogDebug($"Metadata sync from Keyfactor to DigiCert complete. Number of certs synced: {certcounttracker.ToString()}"); + } } } } diff --git a/digicert-metadata-sync/Models/DigicertCustomField.cs b/digicert-metadata-sync/Models/DigicertCustomField.cs new file mode 100644 index 0000000..a622ce7 --- /dev/null +++ b/digicert-metadata-sync/Models/DigicertCustomField.cs @@ -0,0 +1,36 @@ +// Copyright 2021 Keyfactor +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace DigicertMetadataSync; + +partial class DigicertSync +{ + + public class DigicertCustomFieldInstance + { + public int id { get; set; } = 999999999; + public string label { get; set; } = ""; + public bool is_required { get; set; } = false; + public bool is_active { get; set; } = true; + public string data_type { get; set; } = "anything"; + public string kf_field_name { get; set; } = ""; + } + + public class DigicertMetadataUpdateInstance + { + public int metadata_id { get; set; } = 999999999; + public string value { get; set; } = "false"; + } + +} \ No newline at end of file diff --git a/digicert-metadata-sync/Models/KeyfactorCertInstance.cs b/digicert-metadata-sync/Models/KeyfactorCertInstance.cs index 680eeb4..11e3670 100644 --- a/digicert-metadata-sync/Models/KeyfactorCertInstance.cs +++ b/digicert-metadata-sync/Models/KeyfactorCertInstance.cs @@ -66,6 +66,7 @@ public class KeyfactorCert public class Metadata { + } public class Detailedkeyusage diff --git a/readme_source.md b/readme_source.md index 2ff84b7..d287e6a 100644 --- a/readme_source.md +++ b/readme_source.md @@ -1,16 +1,27 @@ + + ## Overview -This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. +This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. The latest update allows for syncronization of custom field contents from Keyfactor to DigiCert. New fields are created in Keyfactor and DigiCert to accomodate for this. ## Installation and Usage The tool comes as a Windows executable. The tool performs synchronization each time its run. For the tool to run automatically, it needs to be added as a scheduled process using Windows. The advised interval for running it is once per week. The files App.config and manualfields.json need to be present in the same directory as the tool for it to run correctly. The specific location from which the tool is ran does not matter, but it needs to have access to both the Keyfactor API endpoint as well as Digicert, and appropriate permissions for access to the configuration files. An explanation for the settings found in these files is given below. +## Command Line Arguments +One of these two arguments needs to be used for the tool to run. +- "kftodc" +Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. +- "dctokf" +Syncronizes the contents of both custom and non-custom fields from DigiCert to Keyfactor. The fields are listed in manualfields.json, and are created if necessary. + ## Settings The settings currently present in these files are shown as an example and need to be configured for your specific situation. ### app.config settings - DigicertAPIKey -Standard DigiCert API access key +Standard DigiCert API access key. +- DigicertAPIKeyTopPerm +DigiCert API access key with restrictions set to "None" - required for sync from Keyfactor to DigiCert. - KeyfactorDomainAndUser Same credential as used when logging into Keyfactor Command. A different set of credentials can be used provided they have adequate access permissions. - KeyfactorPassword @@ -53,3 +64,4 @@ String to be input into Keyfactor as the metadata field hint. - KeyfactorAllowAPI Allows API management of this metadata field in Keyfactor. Should be set to true for continuous synchronization with this tool. + From 7ca9f4bb424e635572af1c6e530579f865fc3490 Mon Sep 17 00:00:00 2001 From: Keyfactor Date: Tue, 28 Mar 2023 19:56:19 +0000 Subject: [PATCH 2/6] Update generated README --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ef850f0..c27f755 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ ___ + + ## Overview This tool primarily sets up metadata fields in Keyfactor for the custom metadata fields in DigiCert, which are named as such, but can also setup metadata fields in Keyfactor for non-custom fields available in DigiCert and unavailable in Keyfactor by default, such as the Digicert Cert ID and the Organization contact. These fields are referred to as manual fields in the context of this tool. After setting up these fields, the tool proceeds to update the contents of these fields. This tool only adds metadata to certificates that have already been imported into Keyfactor. Additionally, this tool requires a properly installed and functioning AnyGateway configured to work with Keyfactor and Digicert. The latest update allows for syncronization of custom field contents from Keyfactor to DigiCert. New fields are created in Keyfactor and DigiCert to accomodate for this. @@ -85,3 +87,4 @@ String to be input into Keyfactor as the metadata field hint. - KeyfactorAllowAPI Allows API management of this metadata field in Keyfactor. Should be set to true for continuous synchronization with this tool. + From 47ca26bd0f21754442e41b21dc135b7c82cc6ff0 Mon Sep 17 00:00:00 2001 From: Mark Kachkaev <37276742+mkachk@users.noreply.github.com> Date: Mon, 24 Apr 2023 10:31:25 -0400 Subject: [PATCH 3/6] Update readme_source.md --- readme_source.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme_source.md b/readme_source.md index d287e6a..9245a2b 100644 --- a/readme_source.md +++ b/readme_source.md @@ -10,10 +10,10 @@ An explanation for the settings found in these files is given below. ## Command Line Arguments One of these two arguments needs to be used for the tool to run. - "kftodc" -Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. +Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. Example: ```.\DigicertMetadataSync.exe dctokf``` - "dctokf" Syncronizes the contents of both custom and non-custom fields from DigiCert to Keyfactor. The fields are listed in manualfields.json, and are created if necessary. - +Example: ```.\DigicertMetadataSync.exe kftodc``` ## Settings The settings currently present in these files are shown as an example and need to be configured for your specific situation. From 996a28c2f90b53bc5be64fd49460ff0fdfa8670d Mon Sep 17 00:00:00 2001 From: Mark Kachkaev <37276742+mkachk@users.noreply.github.com> Date: Mon, 24 Apr 2023 10:31:55 -0400 Subject: [PATCH 4/6] Update readme_source.md --- readme_source.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme_source.md b/readme_source.md index 9245a2b..cdb8bcf 100644 --- a/readme_source.md +++ b/readme_source.md @@ -10,10 +10,10 @@ An explanation for the settings found in these files is given below. ## Command Line Arguments One of these two arguments needs to be used for the tool to run. - "kftodc" -Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. Example: ```.\DigicertMetadataSync.exe dctokf``` +Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. Example: ```.\DigicertMetadataSync.exe kftodc``` - "dctokf" Syncronizes the contents of both custom and non-custom fields from DigiCert to Keyfactor. The fields are listed in manualfields.json, and are created if necessary. -Example: ```.\DigicertMetadataSync.exe kftodc``` +Example: ```.\DigicertMetadataSync.exe dctokf``` ## Settings The settings currently present in these files are shown as an example and need to be configured for your specific situation. From 8c3badb871bc12813ba88a853d5fe267648505bf Mon Sep 17 00:00:00 2001 From: Keyfactor Date: Mon, 24 Apr 2023 14:32:04 +0000 Subject: [PATCH 5/6] Update generated README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c27f755..a859a66 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,10 @@ An explanation for the settings found in these files is given below. ## Command Line Arguments One of these two arguments needs to be used for the tool to run. - "kftodc" -Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. +Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. Example: ```.\DigicertMetadataSync.exe dctokf``` - "dctokf" Syncronizes the contents of both custom and non-custom fields from DigiCert to Keyfactor. The fields are listed in manualfields.json, and are created if necessary. - +Example: ```.\DigicertMetadataSync.exe kftodc``` ## Settings The settings currently present in these files are shown as an example and need to be configured for your specific situation. From cd6f574167dab13b1a6cd146e583ff57ecfca43a Mon Sep 17 00:00:00 2001 From: Keyfactor Date: Mon, 24 Apr 2023 14:32:39 +0000 Subject: [PATCH 6/6] Update generated README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a859a66..7465f0e 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,10 @@ An explanation for the settings found in these files is given below. ## Command Line Arguments One of these two arguments needs to be used for the tool to run. - "kftodc" -Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. Example: ```.\DigicertMetadataSync.exe dctokf``` +Syncronizes the contents of custom fields listed in manualfields.json from Keyfactor to DigiCert. If the fields in manualfields.json do not exist in Keyfactor or DigiCert, they are created first. Example: ```.\DigicertMetadataSync.exe kftodc``` - "dctokf" Syncronizes the contents of both custom and non-custom fields from DigiCert to Keyfactor. The fields are listed in manualfields.json, and are created if necessary. -Example: ```.\DigicertMetadataSync.exe kftodc``` +Example: ```.\DigicertMetadataSync.exe dctokf``` ## Settings The settings currently present in these files are shown as an example and need to be configured for your specific situation.