Skip to content

Commit

Permalink
Enhancements
Browse files Browse the repository at this point in the history 
Add no-resolve to generated Clash rule-set if exists (tindy2013#679).
Do not set CURLOPT_USERAGENT if User-Agent header is provided.
Print curl verbose logs with standard format.
Change thread id in logs to thread name.
Optimize generation of rule contents.
  • Loading branch information
@tindy2013 @littleneko
tindy2013 authored and littleneko committed Feb 26, 2024
1 parent 2aae494 commit 1647ee6
Show file tree
Hide file tree
Showing 5 changed files with 130 additions and 57 deletions.
71 changes: 42 additions & 29 deletions src/generator/config/ruleconvert.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,32 @@ std::string convertRuleset(const std::string &content, int type)
}
}

static std::string transformRuleToCommon(string_view_array &temp, const std::string &input, const std::string &group, bool no_resolve_only = false)
{
temp.clear();
std::string strLine;
split(temp, input, ',');
if(temp.size() < 2)
{
strLine = temp[0];
strLine += ",";
strLine += group;
}
else
{
strLine = temp[0];
strLine += ",";
strLine += temp[1];
strLine += ",";
strLine += group;
if(temp.size() > 2 && (!no_resolve_only || temp[2] == "no-resolve"))
{
strLine += ",";
strLine += temp[2];
}
}
return strLine;
}

void rulesetToClash(YAML::Node &base_rule, std::vector<RulesetContent> &ruleset_content_array, bool overwrite_original_rules, bool new_field_name)
{
Expand All @@ -108,6 +134,7 @@ void rulesetToClash(YAML::Node &base_rule, std::vector<RulesetContent> &ruleset_
if(!overwrite_original_rules && base_rule[field_name].IsDefined())
rules = base_rule[field_name];

std::vector<std::string_view> temp(4);
for(RulesetContent &x : ruleset_content_array)
{
if(global.maxAllowedRules && total_rules > global.maxAllowedRules)
Expand All @@ -124,10 +151,8 @@ void rulesetToClash(YAML::Node &base_rule, std::vector<RulesetContent> &ruleset_
strLine = retrieved_rules.substr(2);
if(startsWith(strLine, "FINAL"))
strLine.replace(0, 5, "MATCH");
strLine += "," + rule_group;
if(count_least(strLine, ',', 3))
strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
allRules.emplace_back(std::move(strLine));
strLine = transformRuleToCommon(temp, strLine, rule_group);
allRules.emplace_back(strLine);
total_rules++;
continue;
}
Expand All @@ -152,11 +177,8 @@ void rulesetToClash(YAML::Node &base_rule, std::vector<RulesetContent> &ruleset_
strLine.erase(strLine.find("//"));
strLine = trimWhitespace(strLine);
}
strLine += "," + rule_group;
if(count_least(strLine, ',', 3))
strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
allRules.emplace_back(std::move(strLine));
//rules.push_back(strLine);
strLine = transformRuleToCommon(temp, strLine, rule_group);
allRules.emplace_back(strLine);
}
}

Expand All @@ -183,6 +205,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector<RulesetContent>
}
base_rule.remove(field_name);

string_view_array temp(4);
for(RulesetContent &x : ruleset_content_array)
{
if(global.maxAllowedRules && total_rules > global.maxAllowedRules)
Expand All @@ -199,9 +222,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector<RulesetContent>
strLine = retrieved_rules.substr(2);
if(startsWith(strLine, "FINAL"))
strLine.replace(0, 5, "MATCH");
strLine += "," + rule_group;
if(count_least(strLine, ',', 3))
strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
strLine = transformRuleToCommon(temp, strLine, rule_group);
output_content += " - " + strLine + "\n";
total_rules++;
continue;
Expand All @@ -227,9 +248,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector<RulesetContent>
strLine.erase(strLine.find("//"));
strLine = trimWhitespace(strLine);
}
strLine += "," + rule_group;
if(count_least(strLine, ',', 3))
strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
strLine = transformRuleToCommon(temp, strLine, rule_group);
output_content += " - " + strLine + "\n";
total_rules++;
}
Expand Down Expand Up @@ -277,6 +296,7 @@ void rulesetToSurge(INIReader &base_rule, std::vector<RulesetContent> &ruleset_c

const std::string rule_match_regex = "^(.*?,.*?)(,.*)(,.*)$";

string_view_array temp(4);
for(RulesetContent &x : ruleset_content_array)
{
if(global.maxAllowedRules && total_rules > global.maxAllowedRules)
Expand All @@ -289,21 +309,17 @@ void rulesetToSurge(INIReader &base_rule, std::vector<RulesetContent> &ruleset_c
strLine = x.rule_content.get().substr(2);
if(strLine == "MATCH")
strLine = "FINAL";
strLine += "," + rule_group;
if(surge_ver == -1 || surge_ver == -2)
{
if(count_least(strLine, ',', 3) && regReplace(strLine, rule_match_regex, "$2") == ",no-resolve")
strLine = regReplace(strLine, rule_match_regex, "$1$3$2");
else
strLine = regReplace(strLine, rule_match_regex, "$1$3");
strLine = transformRuleToCommon(temp, strLine, rule_group, true);
}
else
{
if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT") && count_least(strLine, ',', 3))
strLine = regReplace(strLine, rule_match_regex, "$1$3$2");
if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT"))
strLine = transformRuleToCommon(temp, strLine, rule_group);
}
strLine = replaceAllDistinct(strLine, ",,", ",");
allRules.emplace_back(std::move(strLine));
allRules.emplace_back(strLine);
total_rules++;
continue;
}
Expand Down Expand Up @@ -436,15 +452,12 @@ void rulesetToSurge(INIReader &base_rule, std::vector<RulesetContent> &ruleset_c
{
if(startsWith(strLine, "IP-CIDR6"))
strLine.replace(0, 8, "IP6-CIDR");
if(count_least(strLine, ',', 3) && regReplace(strLine, rule_match_regex, "$2") == ",no-resolve")
strLine = regReplace(strLine, rule_match_regex, "$1$3$2");
else
strLine = regReplace(strLine, rule_match_regex, "$1$3");
strLine = transformRuleToCommon(temp, strLine, rule_group, true);
}
else
{
if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT") && count_least(strLine, ',', 3))
strLine = regReplace(strLine, rule_match_regex, "$1$3$2");
if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT"))
strLine = transformRuleToCommon(temp, strLine, rule_group);
}
allRules.emplace_back(std::move(strLine));
total_rules++;
Expand Down
34 changes: 26 additions & 8 deletions src/generator/template/templates.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector<RulesetContent> &rulese
}
if(!script)
rules.emplace_back("RULE-SET," + rule_name + "," + rule_group);
groups.emplace_back(std::move(rule_name));
groups.emplace_back(rule_name);
continue;
}
if(!remote_path_prefix.empty())
Expand All @@ -398,7 +398,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector<RulesetContent> &rulese
{
if(!script)
rules.emplace_back("RULE-SET," + rule_name + "," + rule_group);
groups.emplace_back(std::move(rule_name));
groups.emplace_back(rule_name);
continue;
}
}
Expand All @@ -419,6 +419,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector<RulesetContent> &rulese
strStrm.clear();
strStrm<<retrieved_rules;
std::string::size_type lineSize;
bool has_no_resolve = false;
while(getline(strStrm, strLine, delimiter))
{
lineSize = strLine.size();
Expand All @@ -441,23 +442,40 @@ int renderClashScript(YAML::Node &base_rule, std::vector<RulesetContent> &rulese
}
else
{
strLine += "," + rule_group;
if(count_least(strLine, ',', 3))
strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2");
rules.emplace_back(std::move(strLine));
vArray = split(strLine, ",");
if(vArray.size() < 2)
{
strLine = vArray[0] + "," + rule_group;
}
else
{
strLine = vArray[0] + "," + vArray[1] + "," + rule_group;
if(vArray.size() > 2)
strLine += "," + vArray[2];
}
rules.emplace_back(strLine);
}
}
else if(!has_domain[rule_name] && (startsWith(strLine, "DOMAIN,") || startsWith(strLine, "DOMAIN-SUFFIX,")))
has_domain[rule_name] = true;
else if(!has_ipcidr[rule_name] && (startsWith(strLine, "IP-CIDR,") || startsWith(strLine, "IP-CIDR6,")))
{
has_ipcidr[rule_name] = true;
if(strLine.find(",no-resolve") != std::string::npos)
has_no_resolve = true;
}
}
if(has_domain[rule_name] && !script)
rules.emplace_back("RULE-SET," + rule_name + "_domain," + rule_group);
if(has_ipcidr[rule_name] && !script)
rules.emplace_back("RULE-SET," + rule_name + "_ipcidr," + rule_group);
{
if(has_no_resolve)
rules.emplace_back("RULE-SET," + rule_name + "_ipcidr," + rule_group + ",no-resolve");
else
rules.emplace_back("RULE-SET," + rule_name + "_ipcidr," + rule_group);
}
if(std::find(groups.begin(), groups.end(), rule_name) == groups.end())
groups.emplace_back(std::move(rule_name));
groups.emplace_back(rule_name);
}
}
for(std::string &x : groups)
Expand Down
58 changes: 44 additions & 14 deletions src/handler/webget.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ class RWLock
RWLock cache_rw_lock;

//std::string user_agent_str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36";
static std::string user_agent_str = "subconverter/" VERSION " cURL/" LIBCURL_VERSION;
static auto user_agent_str = "subconverter/" VERSION " cURL/" LIBCURL_VERSION;

struct curl_progress_data
{
Expand All @@ -113,7 +113,7 @@ static inline void curl_init()

static int writer(char *data, size_t size, size_t nmemb, std::string *writerData)
{
if(writerData == NULL)
if(writerData == nullptr)
return 0;

writerData->append(data, size*nmemb);
Expand All @@ -133,7 +133,7 @@ static int size_checker(void *clientp, curl_off_t dltotal, curl_off_t dlnow, cur
{
if(clientp)
{
curl_progress_data *data = reinterpret_cast<curl_progress_data*>(clientp);
auto *data = reinterpret_cast<curl_progress_data*>(clientp);
if(data->size_limit)
{
if(dlnow > data->size_limit)
Expand All @@ -143,18 +143,44 @@ static int size_checker(void *clientp, curl_off_t dltotal, curl_off_t dlnow, cur
return 0;
}

static int logger(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr)
{
(void)handle;
(void)userptr;
std::string prefix;
switch(type)
{
case CURLINFO_TEXT:
prefix = "CURL_INFO";
break;
case CURLINFO_HEADER_IN:
case CURLINFO_HEADER_OUT:
prefix = "CURL_HEADER";
break;
case CURLINFO_DATA_IN:
case CURLINFO_DATA_OUT:
default:
return 0;
}
std::string content(data, size);
if(content.back() == '\n')
content.pop_back();
writeLog(0, prefix + ": " + content, LOG_LEVEL_VERBOSE);
return 0;
}

static inline void curl_set_common_options(CURL *curl_handle, const char *url, curl_progress_data *data)
{
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, global.logLevel == LOG_LEVEL_VERBOSE ? 1L : 0L);
curl_easy_setopt(curl_handle, CURLOPT_DEBUGFUNCTION, logger);
curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl_handle, CURLOPT_MAXREDIRS, 20L);
curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L);
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, 15L);
//curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent_str.data());
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, "");
if(data)
{
Expand All @@ -170,18 +196,18 @@ static int curlGet(const FetchArgument &argument, FetchResult &result)
{
CURL *curl_handle;
std::string *data = result.content, new_url = argument.url;
struct curl_slist *list = NULL;
defer(curl_slist_free_all(list);)
curl_slist *header_list = nullptr;
defer(curl_slist_free_all(header_list);)
long retVal = 0;

curl_init();

curl_handle = curl_easy_init();
if(argument.proxy.size())
if(!argument.proxy.empty())
{
if(startsWith(argument.proxy, "cors:"))
{
//list = curl_slist_append(list, "X-Requested-With: subconverter " VERSION);
// header_list = curl_slist_append(header_list, "X-Requested-With: subconverter " VERSION);
new_url = argument.proxy.substr(5) + argument.url;
}
else
Expand All @@ -190,16 +216,20 @@ static int curlGet(const FetchArgument &argument, FetchResult &result)
curl_progress_data limit;
limit.size_limit = global.maxAllowedDownloadSize;
curl_set_common_options(curl_handle, new_url.data(), &limit);
list = curl_slist_append(list, "Content-Type: application/json;charset=utf-8");
header_list = curl_slist_append(header_list, "Content-Type: application/json;charset=utf-8");
if(argument.request_headers)
{
for(auto &x : *argument.request_headers)
list = curl_slist_append(list, (x.first + ": " + x.second).data());
{
header_list = curl_slist_append(header_list, (x.first + ": " + x.second).data());
}
if(!argument.request_headers->contains("User-Agent"))
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent_str);
}
//list = curl_slist_append(list, "SubConverter-Request: 1");
//list = curl_slist_append(list, "SubConverter-Version: " VERSION);
if(list)
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list);
// header_list = curl_slist_append(header_list, "SubConverter-Request: 1");
// header_list = curl_slist_append(header_list, "SubConverter-Version: " VERSION);
if(header_list)
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, header_list);

if(result.content)
{
Expand Down
Loading

0 comments on commit 1647ee6

Please sign in to comment.