Skip to content

Commit

Permalink
Fix issue with Crtsh HTTP API causing database corruption.
Browse files Browse the repository at this point in the history 
Crtsh has started putting \n in the name_value field of the JSON payload causing issues in the database when that subdomains where pushed.

Update rust-postgres syntax to match latest version.

Revert 7da5730
  • Loading branch information
@Edu4rdSHL
Edu4rdSHL committed Jan 12, 2020
1 parent 274a966 commit 3ed2cae
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 37 deletions.
67 changes: 32 additions & 35 deletions src/lib.rs
View file
Expand Up @@ -11,7 +11,7 @@ mod get_vars;
mod misc;

use crate::errors::*;
use postgres::{Connection, TlsMode};
use postgres::{Client, NoTls};
use rayon::prelude::*;
use std::{
collections::{HashMap, HashSet},
Expand Down Expand Up @@ -377,8 +377,9 @@ fn get_crtsh_subdomains(url_api_crtsh: &str, quiet_flag: bool) -> Option<HashSet
match data_crtsh.json::<HashSet<SubdomainsCrtsh>>() {
Ok(domains_crtsh) => Some(
domains_crtsh
.into_iter()
.map(|sub| sub.name_value)
.iter()
.flat_map(|sub| sub.name_value.split('\n'))
.map(str::to_owned)
.collect(),
),
Err(e) => {
Expand All @@ -405,8 +406,8 @@ fn get_crtsh_db_subdomains(
if !quiet_flag {
misc::show_searching_msg("Crtsh database")
}
match Connection::connect("postgres://guest@crt.sh:5432/certwatch", TlsMode::None) {
Ok(crtsh_db_client) => match crtsh_db_client.query(&crtsh_db_query, &[]) {
match Client::connect("postgres://guest@crt.sh:5432/certwatch", NoTls) {
Ok(mut crtsh_db_client) => match crtsh_db_client.query(crtsh_db_query, &[]) {
Ok(crtsh_db_subdomains) => Some(
crtsh_db_subdomains
.iter()
Expand All @@ -422,8 +423,7 @@ fn get_crtsh_db_subdomains(
if !quiet_flag {
println!(
"❌ A error has occurred while querying the Crtsh database. Error: {}. Trying the API method...",
e.description()
);
e);
}
get_crtsh_subdomains(&url_api_crtsh, quiet_flag)
}
Expand Down Expand Up @@ -646,27 +646,24 @@ fn subdomains_alerts(args: &mut args::Args) -> Result<()> {
args.subdomains.insert(subdomain);
}
}
let connection: postgres::Connection =
Connection::connect(args.postgres_connection.clone(), TlsMode::None)?;
let mut connection: postgres::Client = Client::connect(&args.postgres_connection, NoTls)?;
let mut discord_parameters = HashMap::new();
let mut slack_parameters = HashMap::new();
let mut telegram_parameters = HashMap::new();
let mut webhooks_data = HashMap::new();
connection.execute(
"CREATE TABLE IF NOT EXISTS subdomains (
id SERIAL PRIMARY KEY,
name VARCHAR NOT NULL UNIQUE
name TEXT NOT NULL UNIQUE
)",
&[],
)?;

let existing_subdomains = connection.query(
&format!(
"SELECT name FROM subdomains WHERE name LIKE '%{}'",
&args.target
),
&[],
)?;
let statement: &str = &format!(
"SELECT name FROM subdomains WHERE name LIKE '%{}'",
&args.target
);
let existing_subdomains = connection.query(statement, &[])?;

let existing_subdomains: HashSet<String> = existing_subdomains
.iter()
Expand Down Expand Up @@ -729,14 +726,17 @@ fn subdomains_alerts(args: &mut args::Args) -> Result<()> {
for (webhook, webhooks_payload) in webhooks_data {
if !webhook.is_empty() {
let response = CLIENT.post(webhook).json(&webhooks_payload).send()?;
if response.status().is_success()
&& !new_subdomains.is_empty()
&& commit_to_db_counter == 0
{
if commit_to_db(&connection, &new_subdomains).is_ok() {
if response.status() == 200 || response.status() == 204 {
if commit_to_db_counter == 0
&& !new_subdomains.is_empty()
&& commit_to_db(
Client::connect(&args.postgres_connection, NoTls)?,
&new_subdomains,
)
.is_ok()
{
commit_to_db_counter += 1
}
} else if response.status().is_success() && new_subdomains.is_empty() {
} else if !args.quiet_flag {
eprintln!(
"\nAn error occurred when Findomain tried to publish the data to the following webhook {}. \nError description: {}",
Expand All @@ -748,8 +748,8 @@ fn subdomains_alerts(args: &mut args::Args) -> Result<()> {
Ok(())
}

fn commit_to_db(conn: &postgres::Connection, new_subdomains: &HashSet<String>) -> Result<()> {
let prepared_transaction = conn.transaction()?;
fn commit_to_db(mut conn: postgres::Client, new_subdomains: &HashSet<String>) -> Result<()> {
let mut prepared_transaction = conn.transaction()?;
for subdomain in new_subdomains {
prepared_transaction.execute("INSERT INTO subdomains (name) VALUES ($1)", &[&subdomain])?;
}
Expand All @@ -764,22 +764,19 @@ fn query_findomain_database(args: &mut args::Args) -> Result<()> {
args.target
)
}
let connection: postgres::Connection =
Connection::connect(args.postgres_connection.clone(), TlsMode::None)?;
let mut connection: postgres::Client = Client::connect(&args.postgres_connection, NoTls)?;
connection.execute(
"CREATE TABLE IF NOT EXISTS subdomains (
id SERIAL PRIMARY KEY,
name VARCHAR NOT NULL UNIQUE
name TEXT NOT NULL UNIQUE
)",
&[],
)?;
let existing_subdomains = connection.query(
&format!(
"SELECT name FROM subdomains WHERE name LIKE '%{}'",
&args.target
),
&[],
)?;
let statement: &str = &format!(
"SELECT name FROM subdomains WHERE name LIKE '%{}'",
&args.target
);
let existing_subdomains = connection.query(statement, &[])?;
args.subdomains = existing_subdomains
.iter()
.map(|row| {
Expand Down
8 changes: 6 additions & 2 deletions src/misc.rs
View file
Expand Up @@ -179,7 +179,11 @@ pub fn return_facebook_token() -> String {

pub fn sanitize_subdomain(base_target: &str, subdomain: &str) -> bool {
!subdomain.is_empty()
&& !subdomain.contains(&['[', ']', '{', '}', '(', ')', '*', '|', ':', '<', '>'][..])
&& !subdomain.contains(
&[
'[', ']', '{', '}', '(', ')', '*', '|', ':', '<', '>', '/', '\\',
][..],
)
&& !subdomain.starts_with('.')
&& subdomain.ends_with(base_target)
}
Expand All @@ -189,7 +193,7 @@ pub fn check_http_response_code(
response: &reqwest::Response,
quiet_flag: bool,
) -> bool {
if response.status().is_success() {
if response.status() == 200 {
true
} else {
if !quiet_flag {
Expand Down

0 comments on commit 3ed2cae

Please sign in to comment.