From be1e99ad797c16c2ac8d6af72f32932f7b5d7668 Mon Sep 17 00:00:00 2001 From: Walter Scheper Date: Sat, 31 Oct 2020 20:42:57 -0400 Subject: [PATCH 01/12] Add generators for URLs and domain names This adds a Generator for RFC 3986 compliant URLs, and two generators for RFC 1035 compliant domain names. Closes #17 Signed-off-by: Walter Scheper --- tld.go | 1518 ++++++++++++++++++++++++++++++++++++++++++ url.go | 125 ++++ url_external_test.go | 67 ++ 3 files changed, 1710 insertions(+) create mode 100644 tld.go create mode 100644 url.go create mode 100644 url_external_test.go diff --git a/tld.go b/tld.go new file mode 100644 index 0000000..1e8a775 --- /dev/null +++ b/tld.go @@ -0,0 +1,1518 @@ +package rapid + +import "strings" + +// sourced from https://data.iana.org/TLD/tlds-alpha-by-domain.txt +// Version 2020103100, Last Updated Sat Oct 31 07:07:01 2020 UTC +const tldsByAlpha = ` +AAA +AARP +ABARTH +ABB +ABBOTT +ABBVIE +ABC +ABLE +ABOGADO +ABUDHABI +AC +ACADEMY +ACCENTURE +ACCOUNTANT +ACCOUNTANTS +ACO +ACTOR +AD +ADAC +ADS +ADULT +AE +AEG +AERO +AETNA +AF +AFAMILYCOMPANY +AFL +AFRICA +AG +AGAKHAN +AGENCY +AI +AIG +AIRBUS +AIRFORCE +AIRTEL +AKDN +AL +ALFAROMEO +ALIBABA +ALIPAY +ALLFINANZ +ALLSTATE +ALLY +ALSACE +ALSTOM +AM +AMAZON +AMERICANEXPRESS +AMERICANFAMILY +AMEX +AMFAM +AMICA +AMSTERDAM +ANALYTICS +ANDROID +ANQUAN +ANZ +AO +AOL +APARTMENTS +APP +APPLE +AQ +AQUARELLE +AR +ARAB +ARAMCO +ARCHI +ARMY +ARPA +ART +ARTE +AS +ASDA +ASIA +ASSOCIATES +AT +ATHLETA +ATTORNEY +AU +AUCTION +AUDI +AUDIBLE +AUDIO +AUSPOST +AUTHOR +AUTO +AUTOS +AVIANCA +AW +AWS +AX +AXA +AZ +AZURE +BA +BABY +BAIDU +BANAMEX +BANANAREPUBLIC +BAND +BANK +BAR +BARCELONA +BARCLAYCARD +BARCLAYS +BAREFOOT +BARGAINS +BASEBALL +BASKETBALL +BAUHAUS +BAYERN +BB +BBC +BBT +BBVA +BCG +BCN +BD +BE +BEATS +BEAUTY +BEER +BENTLEY +BERLIN +BEST +BESTBUY +BET +BF +BG +BH +BHARTI +BI +BIBLE +BID +BIKE +BING +BINGO +BIO +BIZ +BJ +BLACK +BLACKFRIDAY +BLOCKBUSTER +BLOG +BLOOMBERG +BLUE +BM +BMS +BMW +BN +BNPPARIBAS +BO +BOATS +BOEHRINGER +BOFA +BOM +BOND +BOO +BOOK +BOOKING +BOSCH +BOSTIK +BOSTON +BOT +BOUTIQUE +BOX +BR +BRADESCO +BRIDGESTONE +BROADWAY +BROKER +BROTHER +BRUSSELS +BS +BT +BUDAPEST +BUGATTI +BUILD +BUILDERS +BUSINESS +BUY +BUZZ +BV +BW +BY +BZ +BZH +CA +CAB +CAFE +CAL +CALL +CALVINKLEIN +CAM +CAMERA +CAMP +CANCERRESEARCH +CANON +CAPETOWN +CAPITAL +CAPITALONE +CAR +CARAVAN +CARDS +CARE +CAREER +CAREERS +CARS +CASA +CASE +CASEIH +CASH +CASINO +CAT +CATERING +CATHOLIC +CBA +CBN +CBRE +CBS +CC +CD +CEB +CENTER +CEO +CERN +CF +CFA +CFD +CG +CH +CHANEL +CHANNEL +CHARITY +CHASE +CHAT +CHEAP +CHINTAI +CHRISTMAS +CHROME +CHURCH +CI +CIPRIANI +CIRCLE +CISCO +CITADEL +CITI +CITIC +CITY +CITYEATS +CK +CL +CLAIMS +CLEANING +CLICK +CLINIC +CLINIQUE +CLOTHING +CLOUD +CLUB +CLUBMED +CM +CN +CO +COACH +CODES +COFFEE +COLLEGE +COLOGNE +COM +COMCAST +COMMBANK +COMMUNITY +COMPANY +COMPARE +COMPUTER +COMSEC +CONDOS +CONSTRUCTION +CONSULTING +CONTACT +CONTRACTORS +COOKING +COOKINGCHANNEL +COOL +COOP +CORSICA +COUNTRY +COUPON +COUPONS +COURSES +CPA +CR +CREDIT +CREDITCARD +CREDITUNION +CRICKET +CROWN +CRS +CRUISE +CRUISES +CSC +CU +CUISINELLA +CV +CW +CX +CY +CYMRU +CYOU +CZ +DABUR +DAD +DANCE +DATA +DATE +DATING +DATSUN +DAY +DCLK +DDS +DE +DEAL +DEALER +DEALS +DEGREE +DELIVERY +DELL +DELOITTE +DELTA +DEMOCRAT +DENTAL +DENTIST +DESI +DESIGN +DEV +DHL +DIAMONDS +DIET +DIGITAL +DIRECT +DIRECTORY +DISCOUNT +DISCOVER +DISH +DIY +DJ +DK +DM +DNP +DO +DOCS +DOCTOR +DOG +DOMAINS +DOT +DOWNLOAD +DRIVE +DTV +DUBAI +DUCK +DUNLOP +DUPONT +DURBAN +DVAG +DVR +DZ +EARTH +EAT +EC +ECO +EDEKA +EDU +EDUCATION +EE +EG +EMAIL +EMERCK +ENERGY +ENGINEER +ENGINEERING +ENTERPRISES +EPSON +EQUIPMENT +ER +ERICSSON +ERNI +ES +ESQ +ESTATE +ET +ETISALAT +EU +EUROVISION +EUS +EVENTS +EXCHANGE +EXPERT +EXPOSED +EXPRESS +EXTRASPACE +FAGE +FAIL +FAIRWINDS +FAITH +FAMILY +FAN +FANS +FARM +FARMERS +FASHION +FAST +FEDEX +FEEDBACK +FERRARI +FERRERO +FI +FIAT +FIDELITY +FIDO +FILM +FINAL +FINANCE +FINANCIAL +FIRE +FIRESTONE +FIRMDALE +FISH +FISHING +FIT +FITNESS +FJ +FK +FLICKR +FLIGHTS +FLIR +FLORIST +FLOWERS +FLY +FM +FO +FOO +FOOD +FOODNETWORK +FOOTBALL +FORD +FOREX +FORSALE +FORUM +FOUNDATION +FOX +FR +FREE +FRESENIUS +FRL +FROGANS +FRONTDOOR +FRONTIER +FTR +FUJITSU +FUJIXEROX +FUN +FUND +FURNITURE +FUTBOL +FYI +GA +GAL +GALLERY +GALLO +GALLUP +GAME +GAMES +GAP +GARDEN +GAY +GB +GBIZ +GD +GDN +GE +GEA +GENT +GENTING +GEORGE +GF +GG +GGEE +GH +GI +GIFT +GIFTS +GIVES +GIVING +GL +GLADE +GLASS +GLE +GLOBAL +GLOBO +GM +GMAIL +GMBH +GMO +GMX +GN +GODADDY +GOLD +GOLDPOINT +GOLF +GOO +GOODYEAR +GOOG +GOOGLE +GOP +GOT +GOV +GP +GQ +GR +GRAINGER +GRAPHICS +GRATIS +GREEN +GRIPE +GROCERY +GROUP +GS +GT +GU +GUARDIAN +GUCCI +GUGE +GUIDE +GUITARS +GURU +GW +GY +HAIR +HAMBURG +HANGOUT +HAUS +HBO +HDFC +HDFCBANK +HEALTH +HEALTHCARE +HELP +HELSINKI +HERE +HERMES +HGTV +HIPHOP +HISAMITSU +HITACHI +HIV +HK +HKT +HM +HN +HOCKEY +HOLDINGS +HOLIDAY +HOMEDEPOT +HOMEGOODS +HOMES +HOMESENSE +HONDA +HORSE +HOSPITAL +HOST +HOSTING +HOT +HOTELES +HOTELS +HOTMAIL +HOUSE +HOW +HR +HSBC +HT +HU +HUGHES +HYATT +HYUNDAI +IBM +ICBC +ICE +ICU +ID +IE +IEEE +IFM +IKANO +IL +IM +IMAMAT +IMDB +IMMO +IMMOBILIEN +IN +INC +INDUSTRIES +INFINITI +INFO +ING +INK +INSTITUTE +INSURANCE +INSURE +INT +INTERNATIONAL +INTUIT +INVESTMENTS +IO +IPIRANGA +IQ +IR +IRISH +IS +ISMAILI +IST +ISTANBUL +IT +ITAU +ITV +IVECO +JAGUAR +JAVA +JCB +JCP +JE +JEEP +JETZT +JEWELRY +JIO +JLL +JM +JMP +JNJ +JO +JOBS +JOBURG +JOT +JOY +JP +JPMORGAN +JPRS +JUEGOS +JUNIPER +KAUFEN +KDDI +KE +KERRYHOTELS +KERRYLOGISTICS +KERRYPROPERTIES +KFH +KG +KH +KI +KIA +KIM +KINDER +KINDLE +KITCHEN +KIWI +KM +KN +KOELN +KOMATSU +KOSHER +KP +KPMG +KPN +KR +KRD +KRED +KUOKGROUP +KW +KY +KYOTO +KZ +LA +LACAIXA +LAMBORGHINI +LAMER +LANCASTER +LANCIA +LAND +LANDROVER +LANXESS +LASALLE +LAT +LATINO +LATROBE +LAW +LAWYER +LB +LC +LDS +LEASE +LECLERC +LEFRAK +LEGAL +LEGO +LEXUS +LGBT +LI +LIDL +LIFE +LIFEINSURANCE +LIFESTYLE +LIGHTING +LIKE +LILLY +LIMITED +LIMO +LINCOLN +LINDE +LINK +LIPSY +LIVE +LIVING +LIXIL +LK +LLC +LLP +LOAN +LOANS +LOCKER +LOCUS +LOFT +LOL +LONDON +LOTTE +LOTTO +LOVE +LPL +LPLFINANCIAL +LR +LS +LT +LTD +LTDA +LU +LUNDBECK +LUPIN +LUXE +LUXURY +LV +LY +MA +MACYS +MADRID +MAIF +MAISON +MAKEUP +MAN +MANAGEMENT +MANGO +MAP +MARKET +MARKETING +MARKETS +MARRIOTT +MARSHALLS +MASERATI +MATTEL +MBA +MC +MCKINSEY +MD +ME +MED +MEDIA +MEET +MELBOURNE +MEME +MEMORIAL +MEN +MENU +MERCKMSD +MG +MH +MIAMI +MICROSOFT +MIL +MINI +MINT +MIT +MITSUBISHI +MK +ML +MLB +MLS +MM +MMA +MN +MO +MOBI +MOBILE +MODA +MOE +MOI +MOM +MONASH +MONEY +MONSTER +MORMON +MORTGAGE +MOSCOW +MOTO +MOTORCYCLES +MOV +MOVIE +MP +MQ +MR +MS +MSD +MT +MTN +MTR +MU +MUSEUM +MUTUAL +MV +MW +MX +MY +MZ +NA +NAB +NAGOYA +NAME +NATIONWIDE +NATURA +NAVY +NBA +NC +NE +NEC +NET +NETBANK +NETFLIX +NETWORK +NEUSTAR +NEW +NEWHOLLAND +NEWS +NEXT +NEXTDIRECT +NEXUS +NF +NFL +NG +NGO +NHK +NI +NICO +NIKE +NIKON +NINJA +NISSAN +NISSAY +NL +NO +NOKIA +NORTHWESTERNMUTUAL +NORTON +NOW +NOWRUZ +NOWTV +NP +NR +NRA +NRW +NTT +NU +NYC +NZ +OBI +OBSERVER +OFF +OFFICE +OKINAWA +OLAYAN +OLAYANGROUP +OLDNAVY +OLLO +OM +OMEGA +ONE +ONG +ONL +ONLINE +ONYOURSIDE +OOO +OPEN +ORACLE +ORANGE +ORG +ORGANIC +ORIGINS +OSAKA +OTSUKA +OTT +OVH +PA +PAGE +PANASONIC +PARIS +PARS +PARTNERS +PARTS +PARTY +PASSAGENS +PAY +PCCW +PE +PET +PF +PFIZER +PG +PH +PHARMACY +PHD +PHILIPS +PHONE +PHOTO +PHOTOGRAPHY +PHOTOS +PHYSIO +PICS +PICTET +PICTURES +PID +PIN +PING +PINK +PIONEER +PIZZA +PK +PL +PLACE +PLAY +PLAYSTATION +PLUMBING +PLUS +PM +PN +PNC +POHL +POKER +POLITIE +PORN +POST +PR +PRAMERICA +PRAXI +PRESS +PRIME +PRO +PROD +PRODUCTIONS +PROF +PROGRESSIVE +PROMO +PROPERTIES +PROPERTY +PROTECTION +PRU +PRUDENTIAL +PS +PT +PUB +PW +PWC +PY +QA +QPON +QUEBEC +QUEST +QVC +RACING +RADIO +RAID +RE +READ +REALESTATE +REALTOR +REALTY +RECIPES +RED +REDSTONE +REDUMBRELLA +REHAB +REISE +REISEN +REIT +RELIANCE +REN +RENT +RENTALS +REPAIR +REPORT +REPUBLICAN +REST +RESTAURANT +REVIEW +REVIEWS +REXROTH +RICH +RICHARDLI +RICOH +RIL +RIO +RIP +RMIT +RO +ROCHER +ROCKS +RODEO +ROGERS +ROOM +RS +RSVP +RU +RUGBY +RUHR +RUN +RW +RWE +RYUKYU +SA +SAARLAND +SAFE +SAFETY +SAKURA +SALE +SALON +SAMSCLUB +SAMSUNG +SANDVIK +SANDVIKCOROMANT +SANOFI +SAP +SARL +SAS +SAVE +SAXO +SB +SBI +SBS +SC +SCA +SCB +SCHAEFFLER +SCHMIDT +SCHOLARSHIPS +SCHOOL +SCHULE +SCHWARZ +SCIENCE +SCJOHNSON +SCOT +SD +SE +SEARCH +SEAT +SECURE +SECURITY +SEEK +SELECT +SENER +SERVICES +SES +SEVEN +SEW +SEX +SEXY +SFR +SG +SH +SHANGRILA +SHARP +SHAW +SHELL +SHIA +SHIKSHA +SHOES +SHOP +SHOPPING +SHOUJI +SHOW +SHOWTIME +SHRIRAM +SI +SILK +SINA +SINGLES +SITE +SJ +SK +SKI +SKIN +SKY +SKYPE +SL +SLING +SM +SMART +SMILE +SN +SNCF +SO +SOCCER +SOCIAL +SOFTBANK +SOFTWARE +SOHU +SOLAR +SOLUTIONS +SONG +SONY +SOY +SPA +SPACE +SPORT +SPOT +SPREADBETTING +SR +SRL +SS +ST +STADA +STAPLES +STAR +STATEBANK +STATEFARM +STC +STCGROUP +STOCKHOLM +STORAGE +STORE +STREAM +STUDIO +STUDY +STYLE +SU +SUCKS +SUPPLIES +SUPPLY +SUPPORT +SURF +SURGERY +SUZUKI +SV +SWATCH +SWIFTCOVER +SWISS +SX +SY +SYDNEY +SYSTEMS +SZ +TAB +TAIPEI +TALK +TAOBAO +TARGET +TATAMOTORS +TATAR +TATTOO +TAX +TAXI +TC +TCI +TD +TDK +TEAM +TECH +TECHNOLOGY +TEL +TEMASEK +TENNIS +TEVA +TF +TG +TH +THD +THEATER +THEATRE +TIAA +TICKETS +TIENDA +TIFFANY +TIPS +TIRES +TIROL +TJ +TJMAXX +TJX +TK +TKMAXX +TL +TM +TMALL +TN +TO +TODAY +TOKYO +TOOLS +TOP +TORAY +TOSHIBA +TOTAL +TOURS +TOWN +TOYOTA +TOYS +TR +TRADE +TRADING +TRAINING +TRAVEL +TRAVELCHANNEL +TRAVELERS +TRAVELERSINSURANCE +TRUST +TRV +TT +TUBE +TUI +TUNES +TUSHU +TV +TVS +TW +TZ +UA +UBANK +UBS +UG +UK +UNICOM +UNIVERSITY +UNO +UOL +UPS +US +UY +UZ +VA +VACATIONS +VANA +VANGUARD +VC +VE +VEGAS +VENTURES +VERISIGN +VERSICHERUNG +VET +VG +VI +VIAJES +VIDEO +VIG +VIKING +VILLAS +VIN +VIP +VIRGIN +VISA +VISION +VIVA +VIVO +VLAANDEREN +VN +VODKA +VOLKSWAGEN +VOLVO +VOTE +VOTING +VOTO +VOYAGE +VU +VUELOS +WALES +WALMART +WALTER +WANG +WANGGOU +WATCH +WATCHES +WEATHER +WEATHERCHANNEL +WEBCAM +WEBER +WEBSITE +WED +WEDDING +WEIBO +WEIR +WF +WHOSWHO +WIEN +WIKI +WILLIAMHILL +WIN +WINDOWS +WINE +WINNERS +WME +WOLTERSKLUWER +WOODSIDE +WORK +WORKS +WORLD +WOW +WS +WTC +WTF +XBOX +XEROX +XFINITY +XIHUAN +XIN +XN--11B4C3D +XN--1CK2E1B +XN--1QQW23A +XN--2SCRJ9C +XN--30RR7Y +XN--3BST00M +XN--3DS443G +XN--3E0B707E +XN--3HCRJ9C +XN--3OQ18VL8PN36A +XN--3PXU8K +XN--42C2D9A +XN--45BR5CYL +XN--45BRJ9C +XN--45Q11C +XN--4GBRIM +XN--54B7FTA0CC +XN--55QW42G +XN--55QX5D +XN--5SU34J936BGSG +XN--5TZM5G +XN--6FRZ82G +XN--6QQ986B3XL +XN--80ADXHKS +XN--80AO21A +XN--80AQECDR1A +XN--80ASEHDB +XN--80ASWG +XN--8Y0A063A +XN--90A3AC +XN--90AE +XN--90AIS +XN--9DBQ2A +XN--9ET52U +XN--9KRT00A +XN--B4W605FERD +XN--BCK1B9A5DRE4C +XN--C1AVG +XN--C2BR7G +XN--CCK2B3B +XN--CCKWCXETD +XN--CG4BKI +XN--CLCHC0EA0B2G2A9GCD +XN--CZR694B +XN--CZRS0T +XN--CZRU2D +XN--D1ACJ3B +XN--D1ALF +XN--E1A4C +XN--ECKVDTC9D +XN--EFVY88H +XN--FCT429K +XN--FHBEI +XN--FIQ228C5HS +XN--FIQ64B +XN--FIQS8S +XN--FIQZ9S +XN--FJQ720A +XN--FLW351E +XN--FPCRJ9C3D +XN--FZC2C9E2C +XN--FZYS8D69UVGM +XN--G2XX48C +XN--GCKR3F0F +XN--GECRJ9C +XN--GK3AT1E +XN--H2BREG3EVE +XN--H2BRJ9C +XN--H2BRJ9C8C +XN--HXT814E +XN--I1B6B1A6A2E +XN--IMR513N +XN--IO0A7I +XN--J1AEF +XN--J1AMH +XN--J6W193G +XN--JLQ480N2RG +XN--JLQ61U9W7B +XN--JVR189M +XN--KCRX77D1X4A +XN--KPRW13D +XN--KPRY57D +XN--KPUT3I +XN--L1ACC +XN--LGBBAT1AD8J +XN--MGB9AWBF +XN--MGBA3A3EJT +XN--MGBA3A4F16A +XN--MGBA7C0BBN0A +XN--MGBAAKC7DVF +XN--MGBAAM7A8H +XN--MGBAB2BD +XN--MGBAH1A3HJKRD +XN--MGBAI9AZGQP6J +XN--MGBAYH7GPA +XN--MGBBH1A +XN--MGBBH1A71E +XN--MGBC0A9AZCG +XN--MGBCA7DZDO +XN--MGBCPQ6GPA1A +XN--MGBERP4A5D4AR +XN--MGBGU82A +XN--MGBI4ECEXP +XN--MGBPL2FH +XN--MGBT3DHD +XN--MGBTX2B +XN--MGBX4CD0AB +XN--MIX891F +XN--MK1BU44C +XN--MXTQ1M +XN--NGBC5AZD +XN--NGBE9E0A +XN--NGBRX +XN--NODE +XN--NQV7F +XN--NQV7FS00EMA +XN--NYQY26A +XN--O3CW4H +XN--OGBPF8FL +XN--OTU796D +XN--P1ACF +XN--P1AI +XN--PGBS0DH +XN--PSSY2U +XN--Q7CE6A +XN--Q9JYB4C +XN--QCKA1PMC +XN--QXA6A +XN--QXAM +XN--RHQV96G +XN--ROVU88B +XN--RVC1E0AM3E +XN--S9BRJ9C +XN--SES554G +XN--T60B56A +XN--TCKWE +XN--TIQ49XQYJ +XN--UNUP4Y +XN--VERMGENSBERATER-CTB +XN--VERMGENSBERATUNG-PWB +XN--VHQUV +XN--VUQ861B +XN--W4R85EL8FHU5DNRA +XN--W4RS40L +XN--WGBH1C +XN--WGBL6A +XN--XHQ521B +XN--XKC2AL3HYE2A +XN--XKC2DL3A5EE0H +XN--Y9A3AQ +XN--YFRO4I67O +XN--YGBI2AMMX +XN--ZFR164B +XXX +XYZ +YACHTS +YAHOO +YAMAXUN +YANDEX +YE +YODOBASHI +YOGA +YOKOHAMA +YOU +YOUTUBE +YT +YUN +ZA +ZAPPOS +ZARA +ZERO +ZIP +ZM +ZONE +ZUERICH +ZW +` + +var tlds = strings.Split(tldsByAlpha, "\n") diff --git a/url.go b/url.go new file mode 100644 index 0000000..4dba2ff --- /dev/null +++ b/url.go @@ -0,0 +1,125 @@ +package rapid + +import ( + "fmt" + "net/url" + "path" + "reflect" + "strings" + "unicode" +) + +type domainNameGen struct { + maxLength int + maxElementLength int +} + +func (g *domainNameGen) String() string { + return fmt.Sprintf("Domain(maxLength=%v, mmaxElementLength%v)", g.maxLength, g.maxElementLength) +} + +func (g *domainNameGen) type_() reflect.Type { + return stringType +} + +func (g *domainNameGen) value(t *T) value { + domain := SampledFrom(tlds). + Filter(func(s string) bool { return len(s)+2 <= g.maxLength }). + Map(func(s string) string { + var n string + for _, ch := range s { + n += string(SampledFrom([]rune{unicode.ToUpper(ch), unicode.ToLower(ch)}).Draw(t, "").(rune)) + } + + return n + }).Draw(t, "domain").(string) + + var b strings.Builder + elements := newRepeat(1, 126, 1) + b.Grow(elements.avg()) + + var expr string + switch g.maxElementLength { + case 1: + expr = `[a-zA-Z]` + case 2: + expr = `[a-zA-Z][a-zA-Z0-9]?` + default: + expr = fmt.Sprintf(`[a-zA-Z]([a-zA-Z0-9\-]{0,%d}[a-zA-Z0-9])?`, g.maxElementLength-2) + } + for elements.more(t.s, g.String()) { + subDomain := StringMatching(expr).Draw(t, "subdomain").(string) + if len(domain)+len(subDomain) >= g.maxLength { + break + } + domain = subDomain + "." + domain + } + + return domain +} + +// Domain generates an RFC 1035 compliant domain name. +func Domain() *Generator { + return DomainOf(255, 63) +} + +// DomainOf generates an RFC 1035 compliant domain name, +// with a maximum overall length of maxLength +// and a maximum number of elements of maxElements. +func DomainOf(maxLength, maxElementLength int) *Generator { + assertf(4 <= maxLength, "maximum length (%v) should not be less than 4, to generate a two character domain and a one character subdomain", maxLength) + assertf(maxLength <= 255, "maximum length (%v) should not be greater than 255 to comply with RFC 1035", maxLength) + assertf(1 <= maxElementLength, "maximum element length (%v) should not be less than 1 to comply with RFC 1035", maxElementLength) + assertf(maxElementLength <= 63, "maximum element length (%v) should not be greater than 63 to comply with RFC 1035", maxElementLength) + + return newGenerator(&domainNameGen{ + maxElementLength: maxElementLength, + maxLength: maxLength, + }) +} + +type urlGenerator struct { + schemes []string +} + +func (g *urlGenerator) String() string { + return fmt.Sprintf("URLGenerator(schemes=%v)", g.schemes) +} + +func (g *urlGenerator) type_() reflect.Type { + return reflect.TypeOf(url.URL{}) +} + +func (g *urlGenerator) value(t *T) value { + scheme := SampledFrom(g.schemes).Draw(t, "scheme").(string) + domain := Domain().Draw(t, "domain").(string) + port := IntRange(0, 2^16-1). + Map(func(i int) string { + if i == 0 { + return "" + } + return fmt.Sprintf(":%d", i) + }). + Draw(t, "port").(string) + path_ := path.Join( + SliceOf( + StringOf(RuneFrom(nil, unicode.PrintRanges...)).Map(url.PathEscape), + ).Draw(t, "path").([]string)...) + + return url.URL{ + Host: domain + port, + Path: path_, + Scheme: scheme, + } +} + +// URL generates RFC 3986 compliant http/https URLs. +func URL() *Generator { + return urlOf([]string{"http", "https"}) +} + +func urlOf(schemes []string) *Generator { + return newGenerator(&urlGenerator{ + schemes: schemes, + }) +} diff --git a/url_external_test.go b/url_external_test.go new file mode 100644 index 0000000..26fa70d --- /dev/null +++ b/url_external_test.go @@ -0,0 +1,67 @@ +package rapid_test + +import ( + "net/url" + "regexp" + "strconv" + "strings" + "testing" + + . "pgregory.net/rapid" +) + +func TestURL(t *testing.T) { + pathEscapeRegex := regexp.MustCompile(`^[0-9A-Fa-f]{2}`) + + Check(t, func(t *T) { + u := URL().Draw(t, "url").(url.URL) + + // should be parseable + if _, err := url.Parse(u.String()); err != nil { + t.Fatalf("URL returned unparseable url %s: %v", u.String(), err) + } + + // only valid characters in path + for i, ch := range u.Path { + if !(('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z') || ('0' <= ch && ch <= '9') || strings.ContainsRune("$-_.+!*'(),%/@=&:~", ch)) { + t.Fatalf("URL returned invalid url %s: invalid character %s at %d", u.String(), string(ch), i) + } + } + + // assert proper path escapes + for _, co := range strings.Split(u.Path, "%")[1:] { + if ok := pathEscapeRegex.MatchString(co); !ok { + t.Fatalf("URL returned invalid url %s: invalid escape %s", u.String(), co) + } + } + }) +} + +func TestDomainOf(t *testing.T) { + t.Parallel() + + genFuncs := []func(int, int) *Generator{ + func(i, j int) *Generator { return DomainOf(i, j) }, + } + + for i, gf := range genFuncs { + t.Run(strconv.Itoa(i), MakeCheck(func(t *T) { + maxLength := IntRange(4, 255).Draw(t, "maxLength").(int) + maxElementLength := IntRange(1, 63).Draw(t, "maxElementLength").(int) + + d := gf(maxLength, maxElementLength).Draw(t, "d").(string) + if got, want := len(d), maxLength; got > want { + t.Errorf("got domain of length %d with maxLenght of %d", got, want) + } + + elements := strings.Split(d, ".") + + // ignore the tld + for i, elem := range elements[:len(elements)-1] { + if got, want := len(elem), maxElementLength; got > want { + t.Errorf("got domain element %d of length %d with maxElementLength %d", i, got, want) + } + } + })) + } +} From e4d47fda056867ef873c5c2870fd7fe8683b3300 Mon Sep 17 00:00:00 2001 From: Walter Scheper Date: Mon, 2 Nov 2020 19:40:59 -0500 Subject: [PATCH 02/12] Add examples for URL and domain generators Signed-off-by: Walter Scheper --- url_example_test.go | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 url_example_test.go diff --git a/url_example_test.go b/url_example_test.go new file mode 100644 index 0000000..aba6a6a --- /dev/null +++ b/url_example_test.go @@ -0,0 +1,52 @@ +package rapid_test + +import ( + "fmt" + + "pgregory.net/rapid" +) + +func ExampleDomain() { + gen := rapid.Domain() + + for i := 0; i < 5; i++ { + fmt.Println(gen.Example(i)) + } + + // Output: + // MV2zb0-S2j.trAveLcHAnnEL + // Z.CU + // r.ABBotT + // r.AcCoUNTaNT + // R. +} + +func ExampleDomainOf() { + gen := rapid.DomainOf(6, 5) + + for i := 0; i < 5; i++ { + fmt.Println(gen.Example(i)) + } + + // Output: + // Dg5G. + // Z.CU + // Bs. + // AI.HkT + // R. +} + +func ExampleURL() { + gen := rapid.URL() + + for i := 0; i < 5; i++ { + fmt.Println(gen.Example(i)) + } + + // Output: + // {https U.aaA:4 V0%E2%90%9A%226%E0%BC%B0%F0%91%82%B0%F0%97%B3%80%F0%92%91%ADX/1=%22 false } + // {http C.AarP:11 1%EF%BD%9F/%F0%9F%AA%95%22%D6%93%E0%A9%AD%E1%B3%930%D0%8A/%C2%BC%E0%B4%BE3%F0%9E%8B%B0%F0%91%86%BD%C2%B2%E0%B3%A9%CC%80D/%7C+%F0%9F%81%9D+%5D%CC%81%CB%85/%CC%80/%E1%B0%BF/%CD%82K%E0%A5%A9%CC%81 false } + // {https Bs.:11 false } + // {http MC0zJ.aCcOUNtAnT:2 J%E2%9D%87 false } + // {http t.Xn--RvC1e0am3E:3 %CC%82/%E2%80%A60%CC%80%C3%B7/%CC%81%CC%A2%21%E0%AF%AB%CC%81%C3%A4/%F0%9F%AA%95%EA%99%B4%CC%80%E0%A5%AD/%F0%AD%B9%A9%F0%91%87%AE/%E1%B7%93%CC%8B%E2%87%94%E2%90%8E%EA%A3%A5%E0%B5%9A=%E5%8E%A4%D9%AAB%F0%A5%8F%9A=%C2%A4%C3%AE%F0%91%84%AD%DC%8A%21%E2%82%8D3/%E1%81%8F%23 false } +} From 0130cb526d9704614bd0a32d7954e61507f1e180 Mon Sep 17 00:00:00 2001 From: Walter Scheper Date: Mon, 2 Nov 2020 19:43:38 -0500 Subject: [PATCH 03/12] Update copyright headers Signed-off-by: Walter Scheper --- tld.go | 6 ++++++ url.go | 6 ++++++ url_example_test.go | 6 ++++++ url_external_test.go | 6 ++++++ 4 files changed, 24 insertions(+) diff --git a/tld.go b/tld.go index 1e8a775..f107844 100644 --- a/tld.go +++ b/tld.go @@ -1,3 +1,9 @@ +// Copyright 2020 Walter Scheper +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + package rapid import "strings" diff --git a/url.go b/url.go index 4dba2ff..beab283 100644 --- a/url.go +++ b/url.go @@ -1,3 +1,9 @@ +// Copyright 2020 Walter Scheper +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + package rapid import ( diff --git a/url_example_test.go b/url_example_test.go index aba6a6a..cf1da6a 100644 --- a/url_example_test.go +++ b/url_example_test.go @@ -1,3 +1,9 @@ +// Copyright 2020 Walter Scheper +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + package rapid_test import ( diff --git a/url_external_test.go b/url_external_test.go index 26fa70d..1832e73 100644 --- a/url_external_test.go +++ b/url_external_test.go @@ -1,3 +1,9 @@ +// Copyright 2020 Walter Scheper +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + package rapid_test import ( From 131a8ce93d1dda07600abf1264fbfcf6ae8f3cf2 Mon Sep 17 00:00:00 2001 From: Walter Scheper Date: Sun, 8 Nov 2020 20:15:37 -0500 Subject: [PATCH 04/12] Trim tlds list before splitting Need to strip the leading and trailing newlines so we don't get empty strings in our top-level domain list. --- tld.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tld.go b/tld.go index f107844..f3c25b8 100644 --- a/tld.go +++ b/tld.go @@ -1521,4 +1521,4 @@ ZUERICH ZW ` -var tlds = strings.Split(tldsByAlpha, "\n") +var tlds = strings.Split(strings.TrimSpace(tldsByAlpha), "\n") From c8b1ad266bfe150b028155e148619d94abc488f0 Mon Sep 17 00:00:00 2001 From: Walter Scheper Date: Sun, 8 Nov 2020 20:19:08 -0500 Subject: [PATCH 05/12] Simplify domain generation This removes the configurable domain generation in favor of one that generates domains within the full rnage allowed by the specification. --- url.go | 71 +++++++++++++++++--------------------------- url_example_test.go | 39 ++++++++---------------- url_external_test.go | 40 ++++++++++--------------- 3 files changed, 56 insertions(+), 94 deletions(-) diff --git a/url.go b/url.go index beab283..385129f 100644 --- a/url.go +++ b/url.go @@ -11,51 +11,49 @@ import ( "net/url" "path" "reflect" - "strings" "unicode" ) -type domainNameGen struct { - maxLength int - maxElementLength int -} +const ( + domainMaxLength = 255 + domainMaxElementLength = 63 +) + +var ( + domainType = reflect.TypeOf("") + urlType = reflect.TypeOf(url.URL{}) +) + +type domainNameGen struct{} -func (g *domainNameGen) String() string { - return fmt.Sprintf("Domain(maxLength=%v, mmaxElementLength%v)", g.maxLength, g.maxElementLength) +func (*domainNameGen) String() string { + return "Domain()" } -func (g *domainNameGen) type_() reflect.Type { - return stringType +func (*domainNameGen) type_() reflect.Type { + return domainType } +var tldGenerator = SampledFrom(tlds) + func (g *domainNameGen) value(t *T) value { - domain := SampledFrom(tlds). - Filter(func(s string) bool { return len(s)+2 <= g.maxLength }). + domain := tldGenerator. + Filter(func(s string) bool { return len(s)+2 <= domainMaxLength }). Map(func(s string) string { var n string for _, ch := range s { - n += string(SampledFrom([]rune{unicode.ToUpper(ch), unicode.ToLower(ch)}).Draw(t, "").(rune)) + n += string(SampledFrom([]rune{unicode.ToLower(ch), unicode.ToUpper(ch)}).Draw(t, "").(rune)) } return n - }).Draw(t, "domain").(string) + }). + Draw(t, "domain").(string) - var b strings.Builder + expr := fmt.Sprintf(`[a-zA-Z]([a-zA-Z0-9\-]{0,%d}[a-zA-Z0-9])?`, domainMaxElementLength-2) elements := newRepeat(1, 126, 1) - b.Grow(elements.avg()) - - var expr string - switch g.maxElementLength { - case 1: - expr = `[a-zA-Z]` - case 2: - expr = `[a-zA-Z][a-zA-Z0-9]?` - default: - expr = fmt.Sprintf(`[a-zA-Z]([a-zA-Z0-9\-]{0,%d}[a-zA-Z0-9])?`, g.maxElementLength-2) - } for elements.more(t.s, g.String()) { subDomain := StringMatching(expr).Draw(t, "subdomain").(string) - if len(domain)+len(subDomain) >= g.maxLength { + if len(domain)+len(subDomain) >= domainMaxLength { break } domain = subDomain + "." + domain @@ -66,22 +64,7 @@ func (g *domainNameGen) value(t *T) value { // Domain generates an RFC 1035 compliant domain name. func Domain() *Generator { - return DomainOf(255, 63) -} - -// DomainOf generates an RFC 1035 compliant domain name, -// with a maximum overall length of maxLength -// and a maximum number of elements of maxElements. -func DomainOf(maxLength, maxElementLength int) *Generator { - assertf(4 <= maxLength, "maximum length (%v) should not be less than 4, to generate a two character domain and a one character subdomain", maxLength) - assertf(maxLength <= 255, "maximum length (%v) should not be greater than 255 to comply with RFC 1035", maxLength) - assertf(1 <= maxElementLength, "maximum element length (%v) should not be less than 1 to comply with RFC 1035", maxElementLength) - assertf(maxElementLength <= 63, "maximum element length (%v) should not be greater than 63 to comply with RFC 1035", maxElementLength) - - return newGenerator(&domainNameGen{ - maxElementLength: maxElementLength, - maxLength: maxLength, - }) + return newGenerator(&domainNameGen{}) } type urlGenerator struct { @@ -89,11 +72,11 @@ type urlGenerator struct { } func (g *urlGenerator) String() string { - return fmt.Sprintf("URLGenerator(schemes=%v)", g.schemes) + return "URL()" } func (g *urlGenerator) type_() reflect.Type { - return reflect.TypeOf(url.URL{}) + return urlType } func (g *urlGenerator) value(t *T) value { diff --git a/url_example_test.go b/url_example_test.go index cf1da6a..d972997 100644 --- a/url_example_test.go +++ b/url_example_test.go @@ -8,6 +8,7 @@ package rapid_test import ( "fmt" + "net/url" "pgregory.net/rapid" ) @@ -20,39 +21,25 @@ func ExampleDomain() { } // Output: - // MV2zb0-S2j.trAveLcHAnnEL - // Z.CU - // r.ABBotT - // r.AcCoUNTaNT - // R. -} - -func ExampleDomainOf() { - gen := rapid.DomainOf(6, 5) - - for i := 0; i < 5; i++ { - fmt.Println(gen.Example(i)) - } - - // Output: - // Dg5G. - // Z.CU - // Bs. - // AI.HkT - // R. + // D1C.TRaVElErs + // C.cuISiNeLlA + // r.abbVIe + // MC0zJ.aCcOuntAnTs + // T6hFdv10.aaa } func ExampleURL() { gen := rapid.URL() for i := 0; i < 5; i++ { - fmt.Println(gen.Example(i)) + e := gen.Example(i).(url.URL) + fmt.Println(e.String()) } // Output: - // {https U.aaA:4 V0%E2%90%9A%226%E0%BC%B0%F0%91%82%B0%F0%97%B3%80%F0%92%91%ADX/1=%22 false } - // {http C.AarP:11 1%EF%BD%9F/%F0%9F%AA%95%22%D6%93%E0%A9%AD%E1%B3%930%D0%8A/%C2%BC%E0%B4%BE3%F0%9E%8B%B0%F0%91%86%BD%C2%B2%E0%B3%A9%CC%80D/%7C+%F0%9F%81%9D+%5D%CC%81%CB%85/%CC%80/%E1%B0%BF/%CD%82K%E0%A5%A9%CC%81 false } - // {https Bs.:11 false } - // {http MC0zJ.aCcOUNtAnT:2 J%E2%9D%87 false } - // {http t.Xn--RvC1e0am3E:3 %CC%82/%E2%80%A60%CC%80%C3%B7/%CC%81%CC%A2%21%E0%AF%AB%CC%81%C3%A4/%F0%9F%AA%95%EA%99%B4%CC%80%E0%A5%AD/%F0%AD%B9%A9%F0%91%87%AE/%E1%B7%93%CC%8B%E2%87%94%E2%90%8E%EA%A3%A5%E0%B5%9A=%E5%8E%A4%D9%AAB%F0%A5%8F%9A=%C2%A4%C3%AE%F0%91%84%AD%DC%8A%21%E2%82%8D3/%E1%81%8F%23 false } + // https://r125pz05Rz-0j1d-11.AArP:17 + // http://L2.aBArTh:7/%25F0%259F%25AA%2595%2522%25D6%2593%25E0%25A9%25AD%25E1%25B3%25930%25D0%258A/%25C2%25BC%25E0%25B4%25BE3%25F0%259E%258B%25B0%25F0%2591%2586%25BD%25C2%25B2%25E0%25B3%25A9%25CC%2580D/%257C+%25F0%259F%2581%259D+%255D%25CC%2581%25CB%2585/%25CC%2580/%25E1%25B0%25BF/%25CD%2582K%25E0%25A5%25A9%25CC%2581 + // https://pH20DR11.aaA + // http://h.AcCounTaNtS:4/%25F0%259E%25A5%259F/:%2521J%25E2%259D%2587 + // http://A.xN--s9bRJ9C:2 } diff --git a/url_external_test.go b/url_external_test.go index 1832e73..efbaad6 100644 --- a/url_external_test.go +++ b/url_external_test.go @@ -9,15 +9,16 @@ package rapid_test import ( "net/url" "regexp" - "strconv" "strings" "testing" . "pgregory.net/rapid" ) +var pathEscapeRegex = regexp.MustCompile(`^[0-9A-Fa-f]{2}`) + func TestURL(t *testing.T) { - pathEscapeRegex := regexp.MustCompile(`^[0-9A-Fa-f]{2}`) + t.Parallel() Check(t, func(t *T) { u := URL().Draw(t, "url").(url.URL) @@ -43,31 +44,22 @@ func TestURL(t *testing.T) { }) } -func TestDomainOf(t *testing.T) { +func TestDomain(t *testing.T) { t.Parallel() - genFuncs := []func(int, int) *Generator{ - func(i, j int) *Generator { return DomainOf(i, j) }, - } - - for i, gf := range genFuncs { - t.Run(strconv.Itoa(i), MakeCheck(func(t *T) { - maxLength := IntRange(4, 255).Draw(t, "maxLength").(int) - maxElementLength := IntRange(1, 63).Draw(t, "maxElementLength").(int) - - d := gf(maxLength, maxElementLength).Draw(t, "d").(string) - if got, want := len(d), maxLength; got > want { - t.Errorf("got domain of length %d with maxLenght of %d", got, want) - } + Check(t, func(t *T) { + d := Domain().Draw(t, "d").(string) + if got, want := len(d), 255; got > want { + t.Errorf("got domain of length %d with maxLenght of %d", got, want) + } - elements := strings.Split(d, ".") + elements := strings.Split(d, ".") - // ignore the tld - for i, elem := range elements[:len(elements)-1] { - if got, want := len(elem), maxElementLength; got > want { - t.Errorf("got domain element %d of length %d with maxElementLength %d", i, got, want) - } + // ignore the tld + for i, elem := range elements[:len(elements)-1] { + if got, want := len(elem), 63; got > want { + t.Errorf("got domain element %d of length %d with maxElementLength %d", i, got, want) } - })) - } + } + }) } From e7bad3ec4069f99790be735ae79947372a1af528 Mon Sep 17 00:00:00 2001 From: Walter Scheper Date: Tue, 10 Nov 2020 00:11:15 -0500 Subject: [PATCH 06/12] URL generator also generates query and fragment To make the generated URL conform to how `url.Parse` works, Path and Fragment are set to the unescaped form. RawPath and RawFragment are left unset, and we simply rely on the URL struct to handle the escaping as needed. --- url.go | 15 +++++++++------ url_example_test.go | 8 ++++---- url_external_test.go | 17 ----------------- 3 files changed, 13 insertions(+), 27 deletions(-) diff --git a/url.go b/url.go index 385129f..8be1cee 100644 --- a/url.go +++ b/url.go @@ -9,8 +9,8 @@ package rapid import ( "fmt" "net/url" - "path" "reflect" + "strings" "unicode" ) @@ -79,6 +79,8 @@ func (g *urlGenerator) type_() reflect.Type { return urlType } +var printableGen = StringOf(RuneFrom(nil, unicode.PrintRanges...)) + func (g *urlGenerator) value(t *T) value { scheme := SampledFrom(g.schemes).Draw(t, "scheme").(string) domain := Domain().Draw(t, "domain").(string) @@ -90,15 +92,16 @@ func (g *urlGenerator) value(t *T) value { return fmt.Sprintf(":%d", i) }). Draw(t, "port").(string) - path_ := path.Join( - SliceOf( - StringOf(RuneFrom(nil, unicode.PrintRanges...)).Map(url.PathEscape), - ).Draw(t, "path").([]string)...) + path_ := SliceOf(printableGen).Draw(t, "path").([]string) + query := SliceOf(printableGen.Map(url.QueryEscape)).Draw(t, "query").([]string) + fragment := printableGen.Draw(t, "fragment").(string) return url.URL{ Host: domain + port, - Path: path_, + Path: strings.Join(path_, "/"), Scheme: scheme, + RawQuery: strings.Join(query, "&"), + Fragment: fragment, } } diff --git a/url_example_test.go b/url_example_test.go index d972997..6873e6d 100644 --- a/url_example_test.go +++ b/url_example_test.go @@ -37,9 +37,9 @@ func ExampleURL() { } // Output: - // https://r125pz05Rz-0j1d-11.AArP:17 - // http://L2.aBArTh:7/%25F0%259F%25AA%2595%2522%25D6%2593%25E0%25A9%25AD%25E1%25B3%25930%25D0%258A/%25C2%25BC%25E0%25B4%25BE3%25F0%259E%258B%25B0%25F0%2591%2586%25BD%25C2%25B2%25E0%25B3%25A9%25CC%2580D/%257C+%25F0%259F%2581%259D+%255D%25CC%2581%25CB%2585/%25CC%2580/%25E1%25B0%25BF/%25CD%2582K%25E0%25A5%25A9%25CC%2581 - // https://pH20DR11.aaA - // http://h.AcCounTaNtS:4/%25F0%259E%25A5%259F/:%2521J%25E2%259D%2587 + // https://r125pz05Rz-0j1d-11.AArP:17#0%E2%81%9B%F3%A0%84%9A + // http://L2.aBArTh:7/%F0%9F%AA%95%22%D6%93%E0%A9%AD%E1%B3%930%D0%8A/%C2%BC%E0%B4%BE3%F0%9E%8B%B0%F0%91%86%BD%C2%B2%E0%B3%A9%CC%80D/%7C+%F0%9F%81%9D+%5D%CC%81%CB%85/%CC%80/%E1%B0%BF/%CD%82K%E0%A5%A9%CC%81#%CC%82 + // https://pH20DR11.aaA?%DB%97%F0%90%AC%BC%24%F0%91%99%83%E2%82%A5%F0%9D%A8%A8%E0%BC%95%E0%B5%A8%3C%E0%BE%B0%F0%97%8D%91%E2%9E%8E%E0%B9%91%24v%CC%80&%CC%94Z%E4%87%A4#%CC%A0%E1%81%AD + // http://h.AcCounTaNtS:4/%F0%9E%A5%9F/:%21J%E2%9D%87#L%CC%82%E9%98%A6%22 // http://A.xN--s9bRJ9C:2 } diff --git a/url_external_test.go b/url_external_test.go index efbaad6..285f6e3 100644 --- a/url_external_test.go +++ b/url_external_test.go @@ -8,15 +8,12 @@ package rapid_test import ( "net/url" - "regexp" "strings" "testing" . "pgregory.net/rapid" ) -var pathEscapeRegex = regexp.MustCompile(`^[0-9A-Fa-f]{2}`) - func TestURL(t *testing.T) { t.Parallel() @@ -27,20 +24,6 @@ func TestURL(t *testing.T) { if _, err := url.Parse(u.String()); err != nil { t.Fatalf("URL returned unparseable url %s: %v", u.String(), err) } - - // only valid characters in path - for i, ch := range u.Path { - if !(('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z') || ('0' <= ch && ch <= '9') || strings.ContainsRune("$-_.+!*'(),%/@=&:~", ch)) { - t.Fatalf("URL returned invalid url %s: invalid character %s at %d", u.String(), string(ch), i) - } - } - - // assert proper path escapes - for _, co := range strings.Split(u.Path, "%")[1:] { - if ok := pathEscapeRegex.MatchString(co); !ok { - t.Fatalf("URL returned invalid url %s: invalid escape %s", u.String(), co) - } - } }) } From 0a6a87224dcecd27e15f6a0f1b7dfc6e410ae6a0 Mon Sep 17 00:00:00 2001 From: Walter Scheper Date: Tue, 10 Nov 2020 19:57:14 -0500 Subject: [PATCH 07/12] Add IPv4 and IPv6 generators These generators return `net.IP`, and are also used in generating URLs. --- ip_addr.go | 61 +++++++++++++++++++++++++++++++++++++++++ ip_addr_example_test.go | 46 +++++++++++++++++++++++++++++++ url.go | 16 +++++++++-- url_example_test.go | 10 +++---- 4 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 ip_addr.go create mode 100644 ip_addr_example_test.go diff --git a/ip_addr.go b/ip_addr.go new file mode 100644 index 0000000..13e24ec --- /dev/null +++ b/ip_addr.go @@ -0,0 +1,61 @@ +// Copyright 2020 Walter Scheper +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package rapid + +import ( + "fmt" + "net" + "reflect" +) + +const ( + ipv4Len = 4 + ipv6Len = 16 +) + +var ( + ipType = reflect.TypeOf(net.IP{}) + + ipv4Gen = SliceOfN(Byte(), ipv4Len, ipv4Len) + ipv6Gen = SliceOfN(Byte(), ipv6Len, ipv6Len) +) + +type ipGen struct { + ipv6 bool +} + +func (g *ipGen) String() string { + return fmt.Sprintf("IP(ipv6=%v)", g.ipv6) +} + +func (*ipGen) type_() reflect.Type { + return ipType +} + +func (g *ipGen) value(t *T) value { + var gen *Generator + if g.ipv6 { + gen = ipv6Gen + } else { + gen = ipv4Gen + } + + b := gen.Draw(t, g.String()).([]byte) + return net.IP(b) +} + +func IPv4() *Generator { + return newGenerator(&ipGen{ + ipv6: false, + }) +} + +func IPv6() *Generator { + return newGenerator(&ipGen{ + ipv6: true, + }) +} diff --git a/ip_addr_example_test.go b/ip_addr_example_test.go new file mode 100644 index 0000000..1379835 --- /dev/null +++ b/ip_addr_example_test.go @@ -0,0 +1,46 @@ +// Copyright 2020 Walter Scheper +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package rapid_test + +import ( + "fmt" + "net" + + "pgregory.net/rapid" +) + +func ExampleIPv4() { + gen := rapid.IPv4() + + for i := 0; i < 5; i++ { + addr := gen.Example(i).(net.IP) + fmt.Println(addr.String()) + } + + // Output: + // 0.23.24.7 + // 100.146.0.0 + // 0.222.65.1 + // 1.49.104.14 + // 11.56.0.83 +} + +func ExampleIPv6() { + gen := rapid.IPv6() + + for i := 0; i < 5; i++ { + addr := gen.Example(i).(net.IP) + fmt.Println(addr.String()) + } + + // Output: + // 17:1807:e2c4:8210:7202:f4b2:a0e2:8dc + // 6492:0:fa37:b00:b5c3:4e6:6a01:c802 + // de:4101:9f5:3:104:5dc:b600:905 + // 131:680e:97ff:d200:ae1:4d00:2300:103 + // b38:53:ff07:200:8c28:ee:ad00:1b +} diff --git a/url.go b/url.go index 8be1cee..27fb5cd 100644 --- a/url.go +++ b/url.go @@ -8,6 +8,7 @@ package rapid import ( "fmt" + "net" "net/url" "reflect" "strings" @@ -83,7 +84,16 @@ var printableGen = StringOf(RuneFrom(nil, unicode.PrintRanges...)) func (g *urlGenerator) value(t *T) value { scheme := SampledFrom(g.schemes).Draw(t, "scheme").(string) - domain := Domain().Draw(t, "domain").(string) + var domain string + switch SampledFrom([]int{0, 1, 2}).Draw(t, "g").(int) { + case 2: + domain = Domain().Draw(t, "domain").(string) + case 1: + domain = IPv6().Draw(t, "domain").(net.IP).String() + domain = "[" + domain + "]" + case 0: + domain = IPv4().Draw(t, "domain").(net.IP).String() + } port := IntRange(0, 2^16-1). Map(func(i int) string { if i == 0 { @@ -97,9 +107,9 @@ func (g *urlGenerator) value(t *T) value { fragment := printableGen.Draw(t, "fragment").(string) return url.URL{ - Host: domain + port, + Host: domain + port, Path: strings.Join(path_, "/"), - Scheme: scheme, + Scheme: scheme, RawQuery: strings.Join(query, "&"), Fragment: fragment, } diff --git a/url_example_test.go b/url_example_test.go index 6873e6d..8b5aabb 100644 --- a/url_example_test.go +++ b/url_example_test.go @@ -37,9 +37,9 @@ func ExampleURL() { } // Output: - // https://r125pz05Rz-0j1d-11.AArP:17#0%E2%81%9B%F3%A0%84%9A - // http://L2.aBArTh:7/%F0%9F%AA%95%22%D6%93%E0%A9%AD%E1%B3%930%D0%8A/%C2%BC%E0%B4%BE3%F0%9E%8B%B0%F0%91%86%BD%C2%B2%E0%B3%A9%CC%80D/%7C+%F0%9F%81%9D+%5D%CC%81%CB%85/%CC%80/%E1%B0%BF/%CD%82K%E0%A5%A9%CC%81#%CC%82 - // https://pH20DR11.aaA?%DB%97%F0%90%AC%BC%24%F0%91%99%83%E2%82%A5%F0%9D%A8%A8%E0%BC%95%E0%B5%A8%3C%E0%BE%B0%F0%97%8D%91%E2%9E%8E%E0%B9%91%24v%CC%80&%CC%94Z%E4%87%A4#%CC%A0%E1%81%AD - // http://h.AcCounTaNtS:4/%F0%9E%A5%9F/:%21J%E2%9D%87#L%CC%82%E9%98%A6%22 - // http://A.xN--s9bRJ9C:2 + // https://[e506:816b:407:316:fb4c:ffa0:e208:dc0e]/%F0%97%B3%80%F0%92%91%ADX/1=%22?%C4%90%F0%90%A9%87%26#%F0%91%B2%9E1%CC%88%CC%81D + // http://G.BLoG/%E0%AD%8C~%F0%9F%AA%95%22%D6%93%E0%A9%AD%E1%B3%930%D0%8A/%C2%BC%E0%B4%BE3%F0%9E%8B%B0%F0%91%86%BD%C2%B2%E0%B3%A9%CC%80D/%7C+%F0%9F%81%9D+%5D%CC%81%CB%85/%CC%80/%E1%B0%BF/%CD%82K%E0%A5%A9%CC%81#%CC%82 + // https://1.47.4.5:11/+%3E%E2%9F%BCK//A%DB%97%F0%90%AC%BC$%F0%91%99%83%E2%82%A5%F0%9D%A8%A8%E0%BC%95%E0%B5%A8%3C%E0%BE%B0%F0%97%8D%91%E2%9E%8E%E0%B9%91$v%CC%80/%CC%94Z%E4%87%A4?%F0%91%91%9BC%C2%B9%E2%8A%A5%F0%91%91%8F1%E0%A0%BE%CB%BE%C3%9D%E1%B3%A8%E0%AB%A6%CC%81%CC%86&%E2%A4%88%F0%91%8A%A9%24B%F0%9D%8B%AA%CC%9A&&%CC%80%C2%A7%E4%92%9B&#%E0%AB%AE%F0%92%91%A2 + // http://G.hM/%CC%80%E0%A0%B1%CC%82%CC%80%F0%9E%A5%9F/:%21J%E2%9D%87#L%CC%82%E9%98%A6%22 + // http://1.1.4.6:2/%E3%B5%B8%C3%B2%DE%A6%E1%9E%B9/%C2%A7/?%E2%80%A60%CC%80%C3%B7&%CC%81%CC%A2%21%E0%AF%AB%CC%81%C3%A4&%F0%9F%AA%95%EA%99%B4%CC%80%E0%A5%AD&%F0%AD%B9%A9%F0%91%87%AE&&%E1%B7%93%CC%8B%E2%87%94%E2%90%8E%EA%A3%A5%E0%B5%9A%3D%E5%8E%A4%D9%AAB%F0%A5%8F%9A%3D%C2%A4%C3%AE%F0%91%84%AD%DC%8A%21%E2%82%8D3&%E1%81%8F%23#%CC%80%E0%BF%8B+$ } From 18ad72310a0dd33ce32cc34a8024123a121b590c Mon Sep 17 00:00:00 2001 From: Gregory Petrosyan Date: Wed, 10 Mar 2021 01:13:00 +0300 Subject: [PATCH 08/12] Fix URL example tests --- url_example_test.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/url_example_test.go b/url_example_test.go index 8b5aabb..e5acbfd 100644 --- a/url_example_test.go +++ b/url_example_test.go @@ -4,6 +4,9 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +// String generation depends on the Unicode tables, which change with Go versions: +// +build go1.16 + package rapid_test import ( @@ -37,9 +40,9 @@ func ExampleURL() { } // Output: - // https://[e506:816b:407:316:fb4c:ffa0:e208:dc0e]/%F0%97%B3%80%F0%92%91%ADX/1=%22?%C4%90%F0%90%A9%87%26#%F0%91%B2%9E1%CC%88%CC%81D - // http://G.BLoG/%E0%AD%8C~%F0%9F%AA%95%22%D6%93%E0%A9%AD%E1%B3%930%D0%8A/%C2%BC%E0%B4%BE3%F0%9E%8B%B0%F0%91%86%BD%C2%B2%E0%B3%A9%CC%80D/%7C+%F0%9F%81%9D+%5D%CC%81%CB%85/%CC%80/%E1%B0%BF/%CD%82K%E0%A5%A9%CC%81#%CC%82 - // https://1.47.4.5:11/+%3E%E2%9F%BCK//A%DB%97%F0%90%AC%BC$%F0%91%99%83%E2%82%A5%F0%9D%A8%A8%E0%BC%95%E0%B5%A8%3C%E0%BE%B0%F0%97%8D%91%E2%9E%8E%E0%B9%91$v%CC%80/%CC%94Z%E4%87%A4?%F0%91%91%9BC%C2%B9%E2%8A%A5%F0%91%91%8F1%E0%A0%BE%CB%BE%C3%9D%E1%B3%A8%E0%AB%A6%CC%81%CC%86&%E2%A4%88%F0%91%8A%A9%24B%F0%9D%8B%AA%CC%9A&&%CC%80%C2%A7%E4%92%9B&#%E0%AB%AE%F0%92%91%A2 - // http://G.hM/%CC%80%E0%A0%B1%CC%82%CC%80%F0%9E%A5%9F/:%21J%E2%9D%87#L%CC%82%E9%98%A6%22 - // http://1.1.4.6:2/%E3%B5%B8%C3%B2%DE%A6%E1%9E%B9/%C2%A7/?%E2%80%A60%CC%80%C3%B7&%CC%81%CC%A2%21%E0%AF%AB%CC%81%C3%A4&%F0%9F%AA%95%EA%99%B4%CC%80%E0%A5%AD&%F0%AD%B9%A9%F0%91%87%AE&&%E1%B7%93%CC%8B%E2%87%94%E2%90%8E%EA%A3%A5%E0%B5%9A%3D%E5%8E%A4%D9%AAB%F0%A5%8F%9A%3D%C2%A4%C3%AE%F0%91%84%AD%DC%8A%21%E2%82%8D3&%E1%81%8F%23#%CC%80%E0%BF%8B+$ + // https://[e506:816b:407:316:fb4c:ffa0:e208:dc0e]/%F0%97%B0%A0%F0%92%91%9CX/1=%22?%C4%90%F0%90%A9%87%26#%F0%96%AC%B21%CC%88%CC%81D + // http://G.BLoG/%E0%AD%8C~%F0%9F%AF%8A%22%D6%93%E0%A9%AD%E1%B3%930%D0%8A/%C2%BC%E0%B4%BC3%F0%9D%9F%B9%F0%91%91%82%C2%B2%E0%B3%A9%CC%80D/%7C+%F0%9F%82%92+%5D%CC%81%CB%85/%CC%80/%E1%B0%BF/%CD%82K%E0%A5%A9%CC%81#%CC%82 + // https://1.47.4.5:11/+%3E%E2%9F%BCK//A%DB%97%F0%91%99%83$%E0%A0%BD%E2%82%A5%F0%9D%A9%A9%E0%BC%95%E0%B5%A8%3C%E0%BE%AE%F0%97%8A%B1%E2%9E%8E%E0%B9%91$v%CC%80/%CC%94Z%E4%87%94?%F0%96%A9%AEC%C2%B9%E2%8A%A5%F0%92%91%B41%E0%A0%BE%CB%BE%C3%9D%E1%B3%A4%E0%AB%A6%CC%81%CC%86&%E2%A4%88%F0%91%BF%BF%24B%F0%96%BA%90%CC%9A&&%CC%80%C2%A7%E8%93%8B&#%E0%AB%AE%F0%92%91%91 + // http://G.hM/%CC%80%E0%A0%B1%CC%82%CC%80%F0%9E%A5%9F/:%21J%E2%9D%87#L%CC%82%E9%98%8C%22 + // http://1.1.4.6:2/%F0%A7%A8%A4%F0%A1%AD%8D%E2%92%8B0/%DC%B4B?%E2%80%A60%CC%80%C3%B7&%CC%81%CC%A2%21%E0%AF%AB%CC%81%C3%A4&%F0%9F%AF%8A%EA%99%AF%CC%80%E0%A5%AD&%E5%8B%B71&%E1%B7%8F%CC%8B%E2%87%94%E2%90%8E%EA%A3%A0%E0%B5%9A%3D%E5%8E%8A%D9%AAB%F0%A8%83%A2%EF%B8%B4%E0%A0%BD%F0%9D%84%86%C6%81%211A3&%E1%81%8F%23#%CC%80%E0%BF%8B+$ } From 4846a6a1db67138c6b3387032c0f7731c1dd0028 Mon Sep 17 00:00:00 2001 From: Walter Scheper Date: Wed, 1 Sep 2021 22:57:34 -0400 Subject: [PATCH 09/12] Add script to update top-level domains This adds a simple shell script to update the tld.go constant with the latest contents of https://data.iana.org/TLD/tlds-alpha-by-domain.txt --- tld.go | 14 ++------------ update_tld.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 12 deletions(-) create mode 100755 update_tld.sh diff --git a/tld.go b/tld.go index f3c25b8..aa9bc46 100644 --- a/tld.go +++ b/tld.go @@ -9,7 +9,7 @@ package rapid import "strings" // sourced from https://data.iana.org/TLD/tlds-alpha-by-domain.txt -// Version 2020103100, Last Updated Sat Oct 31 07:07:01 2020 UTC +// Version 2021090100, Last Updated Thu Sep 2 02:57:25 UTC 2021 const tldsByAlpha = ` AAA AARP @@ -224,7 +224,6 @@ CAREERS CARS CASA CASE -CASEIH CASH CASINO CAT @@ -236,7 +235,6 @@ CBRE CBS CC CD -CEB CENTER CEO CERN @@ -474,7 +472,6 @@ FRONTDOOR FRONTIER FTR FUJITSU -FUJIXEROX FUN FUND FURNITURE @@ -640,11 +637,9 @@ ISTANBUL IT ITAU ITV -IVECO JAGUAR JAVA JCB -JCP JE JEEP JETZT @@ -760,7 +755,6 @@ LTD LTDA LU LUNDBECK -LUPIN LUXE LUXURY LV @@ -849,7 +843,6 @@ NA NAB NAGOYA NAME -NATIONWIDE NATURA NAVY NBA @@ -862,7 +855,6 @@ NETFLIX NETWORK NEUSTAR NEW -NEWHOLLAND NEWS NEXT NEXTDIRECT @@ -910,7 +902,6 @@ ONE ONG ONL ONLINE -ONYOURSIDE OOO OPEN ORACLE @@ -1110,7 +1101,6 @@ SHOPPING SHOUJI SHOW SHOWTIME -SHRIRAM SI SILK SINA @@ -1144,7 +1134,6 @@ SPA SPACE SPORT SPOT -SPREADBETTING SR SRL SS @@ -1358,6 +1347,7 @@ XN--42C2D9A XN--45BR5CYL XN--45BRJ9C XN--45Q11C +XN--4DBRK0CE XN--4GBRIM XN--54B7FTA0CC XN--55QW42G diff --git a/update_tld.sh b/update_tld.sh new file mode 100755 index 0000000..b6b6186 --- /dev/null +++ b/update_tld.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -eu + +TLD_URL=https://data.iana.org/TLD/tlds-alpha-by-domain.txt +TLD="$(curl -fsSL https://data.iana.org/TLD/tlds-alpha-by-domain.txt | grep -v '^#')" + +cat > tld.go << EOF +// Copyright 2020 Walter Scheper +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package rapid + +import "strings" + +// sourced from https://data.iana.org/TLD/tlds-alpha-by-domain.txt +// Version $(date +%Y%m%d00), Last Updated $(date --utc) +const tldsByAlpha = \` +${TLD} +\` + +var tlds = strings.Split(strings.TrimSpace(tldsByAlpha), "\n") +EOF From 1499acecf6e11e39f6799b680b7e944e1b553116 Mon Sep 17 00:00:00 2001 From: Walter Scheper Date: Sat, 24 Sep 2022 22:10:12 -0400 Subject: [PATCH 10/12] Update to use new generic API --- ip_addr.go | 23 ++++-------- ip_addr_example_test.go | 5 +-- tld.go | 17 ++------- update_tld.sh | 2 +- url.go | 81 ++++++++++++++++++----------------------- url_example_test.go | 24 ++++++------ url_external_test.go | 4 +- 7 files changed, 64 insertions(+), 92 deletions(-) diff --git a/ip_addr.go b/ip_addr.go index 13e24ec..a0f50a3 100644 --- a/ip_addr.go +++ b/ip_addr.go @@ -1,4 +1,4 @@ -// Copyright 2020 Walter Scheper +// Copyright 2022 Walter Scheper // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -9,7 +9,6 @@ package rapid import ( "fmt" "net" - "reflect" ) const ( @@ -18,8 +17,6 @@ const ( ) var ( - ipType = reflect.TypeOf(net.IP{}) - ipv4Gen = SliceOfN(Byte(), ipv4Len, ipv4Len) ipv6Gen = SliceOfN(Byte(), ipv6Len, ipv6Len) ) @@ -32,30 +29,26 @@ func (g *ipGen) String() string { return fmt.Sprintf("IP(ipv6=%v)", g.ipv6) } -func (*ipGen) type_() reflect.Type { - return ipType -} - -func (g *ipGen) value(t *T) value { - var gen *Generator +func (g *ipGen) value(t *T) net.IP { + var gen *Generator[[]byte] if g.ipv6 { gen = ipv6Gen } else { gen = ipv4Gen } - b := gen.Draw(t, g.String()).([]byte) + b := gen.Draw(t, g.String()) return net.IP(b) } -func IPv4() *Generator { - return newGenerator(&ipGen{ +func IPv4() *Generator[net.IP] { + return newGenerator[net.IP](&ipGen{ ipv6: false, }) } -func IPv6() *Generator { - return newGenerator(&ipGen{ +func IPv6() *Generator[net.IP] { + return newGenerator[net.IP](&ipGen{ ipv6: true, }) } diff --git a/ip_addr_example_test.go b/ip_addr_example_test.go index 1379835..8839d42 100644 --- a/ip_addr_example_test.go +++ b/ip_addr_example_test.go @@ -8,7 +8,6 @@ package rapid_test import ( "fmt" - "net" "pgregory.net/rapid" ) @@ -17,7 +16,7 @@ func ExampleIPv4() { gen := rapid.IPv4() for i := 0; i < 5; i++ { - addr := gen.Example(i).(net.IP) + addr := gen.Example(i) fmt.Println(addr.String()) } @@ -33,7 +32,7 @@ func ExampleIPv6() { gen := rapid.IPv6() for i := 0; i < 5; i++ { - addr := gen.Example(i).(net.IP) + addr := gen.Example(i) fmt.Println(addr.String()) } diff --git a/tld.go b/tld.go index aa9bc46..d3774ea 100644 --- a/tld.go +++ b/tld.go @@ -9,7 +9,7 @@ package rapid import "strings" // sourced from https://data.iana.org/TLD/tlds-alpha-by-domain.txt -// Version 2021090100, Last Updated Thu Sep 2 02:57:25 UTC 2021 +// Version 2022092400, Last Updated Sun Sep 25 02:07:59 UTC 2022 const tldsByAlpha = ` AAA AARP @@ -37,7 +37,6 @@ AEG AERO AETNA AF -AFAMILYCOMPANY AFL AFRICA AG @@ -189,7 +188,6 @@ BROTHER BRUSSELS BS BT -BUDAPEST BUGATTI BUILD BUILDERS @@ -313,7 +311,6 @@ CROWN CRS CRUISE CRUISES -CSC CU CUISINELLA CV @@ -372,7 +369,6 @@ DOWNLOAD DRIVE DTV DUBAI -DUCK DUNLOP DUPONT DURBAN @@ -506,7 +502,6 @@ GIFTS GIVES GIVING GL -GLADE GLASS GLE GLOBAL @@ -670,6 +665,7 @@ KG KH KI KIA +KIDS KIM KINDER KINDLE @@ -732,7 +728,6 @@ LINK LIPSY LIVE LIVING -LIXIL LK LLC LLP @@ -833,6 +828,7 @@ MTN MTR MU MUSEUM +MUSIC MUTUAL MV MW @@ -889,7 +885,6 @@ NYC NZ OBI OBSERVER -OFF OFFICE OKINAWA OLAYAN @@ -988,10 +983,8 @@ QA QPON QUEBEC QUEST -QVC RACING RADIO -RAID RE READ REALESTATE @@ -1023,7 +1016,6 @@ RICOH RIL RIO RIP -RMIT RO ROCHER ROCKS @@ -1069,7 +1061,6 @@ SCHOOL SCHULE SCHWARZ SCIENCE -SCJOHNSON SCOT SD SE @@ -1162,7 +1153,6 @@ SURGERY SUZUKI SV SWATCH -SWIFTCOVER SWISS SX SY @@ -1341,7 +1331,6 @@ XN--3BST00M XN--3DS443G XN--3E0B707E XN--3HCRJ9C -XN--3OQ18VL8PN36A XN--3PXU8K XN--42C2D9A XN--45BR5CYL diff --git a/update_tld.sh b/update_tld.sh index b6b6186..779320b 100755 --- a/update_tld.sh +++ b/update_tld.sh @@ -17,7 +17,7 @@ package rapid import "strings" // sourced from https://data.iana.org/TLD/tlds-alpha-by-domain.txt -// Version $(date +%Y%m%d00), Last Updated $(date --utc) +// Version $(date +%Y%m%d00), Last Updated $(date -u) const tldsByAlpha = \` ${TLD} \` diff --git a/url.go b/url.go index 27fb5cd..fa954b0 100644 --- a/url.go +++ b/url.go @@ -8,7 +8,6 @@ package rapid import ( "fmt" - "net" "net/url" "reflect" "strings" @@ -31,29 +30,17 @@ func (*domainNameGen) String() string { return "Domain()" } -func (*domainNameGen) type_() reflect.Type { - return domainType -} - var tldGenerator = SampledFrom(tlds) -func (g *domainNameGen) value(t *T) value { +func (g *domainNameGen) value(t *T) string { domain := tldGenerator. Filter(func(s string) bool { return len(s)+2 <= domainMaxLength }). - Map(func(s string) string { - var n string - for _, ch := range s { - n += string(SampledFrom([]rune{unicode.ToLower(ch), unicode.ToUpper(ch)}).Draw(t, "").(rune)) - } - - return n - }). - Draw(t, "domain").(string) + Draw(t, "domain") expr := fmt.Sprintf(`[a-zA-Z]([a-zA-Z0-9\-]{0,%d}[a-zA-Z0-9])?`, domainMaxElementLength-2) - elements := newRepeat(1, 126, 1) - for elements.more(t.s, g.String()) { - subDomain := StringMatching(expr).Draw(t, "subdomain").(string) + elements := newRepeat(1, 126, 1, g.String()) + for elements.more(t.s) { + subDomain := StringMatching(expr).Draw(t, "subdomain") if len(domain)+len(subDomain) >= domainMaxLength { break } @@ -64,8 +51,8 @@ func (g *domainNameGen) value(t *T) value { } // Domain generates an RFC 1035 compliant domain name. -func Domain() *Generator { - return newGenerator(&domainNameGen{}) +func Domain() *Generator[string] { + return newGenerator[string](&domainNameGen{}) } type urlGenerator struct { @@ -76,38 +63,42 @@ func (g *urlGenerator) String() string { return "URL()" } -func (g *urlGenerator) type_() reflect.Type { - return urlType -} - var printableGen = StringOf(RuneFrom(nil, unicode.PrintRanges...)) -func (g *urlGenerator) value(t *T) value { - scheme := SampledFrom(g.schemes).Draw(t, "scheme").(string) +func (g *urlGenerator) value(t *T) url.URL { + scheme := SampledFrom(g.schemes).Draw(t, "scheme") var domain string - switch SampledFrom([]int{0, 1, 2}).Draw(t, "g").(int) { + switch SampledFrom([]int{0, 1, 2}).Draw(t, "g") { case 2: - domain = Domain().Draw(t, "domain").(string) + domain = Domain().Draw(t, "domain") case 1: - domain = IPv6().Draw(t, "domain").(net.IP).String() + domain = IPv6().Draw(t, "domain").String() domain = "[" + domain + "]" case 0: - domain = IPv4().Draw(t, "domain").(net.IP).String() + domain = IPv4().Draw(t, "domain").String() + } + port := IntRange(0, 2^16-1).Draw(t, "port") + path_ := SliceOf(printableGen).Draw(t, "path") + query := SliceOf(printableGen).Draw(t, "query") + fragment := printableGen.Draw(t, "fragment") + + // join domain and port + if port > 0 { + domain += fmt.Sprintf(":%d", port) + } + + // URL escape path + for i := range path_ { + path_[i] = url.PathEscape(path_[i]) + } + + // url escape query strings + for i := range query { + query[i] = url.QueryEscape(query[i]) } - port := IntRange(0, 2^16-1). - Map(func(i int) string { - if i == 0 { - return "" - } - return fmt.Sprintf(":%d", i) - }). - Draw(t, "port").(string) - path_ := SliceOf(printableGen).Draw(t, "path").([]string) - query := SliceOf(printableGen.Map(url.QueryEscape)).Draw(t, "query").([]string) - fragment := printableGen.Draw(t, "fragment").(string) return url.URL{ - Host: domain + port, + Host: domain, Path: strings.Join(path_, "/"), Scheme: scheme, RawQuery: strings.Join(query, "&"), @@ -116,12 +107,12 @@ func (g *urlGenerator) value(t *T) value { } // URL generates RFC 3986 compliant http/https URLs. -func URL() *Generator { +func URL() *Generator[url.URL] { return urlOf([]string{"http", "https"}) } -func urlOf(schemes []string) *Generator { - return newGenerator(&urlGenerator{ +func urlOf(schemes []string) *Generator[url.URL] { + return newGenerator[url.URL](&urlGenerator{ schemes: schemes, }) } diff --git a/url_example_test.go b/url_example_test.go index e5acbfd..a6ad37c 100644 --- a/url_example_test.go +++ b/url_example_test.go @@ -5,13 +5,13 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. // String generation depends on the Unicode tables, which change with Go versions: +//go:build go1.16 // +build go1.16 package rapid_test import ( "fmt" - "net/url" "pgregory.net/rapid" ) @@ -24,25 +24,25 @@ func ExampleDomain() { } // Output: - // D1C.TRaVElErs - // C.cuISiNeLlA - // r.abbVIe - // MC0zJ.aCcOuntAnTs - // T6hFdv10.aaa + // l.UOL + // S.CYMRU + // e.ABBVIE + // x.ACCOUNTANTS + // R.AAA } func ExampleURL() { gen := rapid.URL() for i := 0; i < 5; i++ { - e := gen.Example(i).(url.URL) + e := gen.Example(i) fmt.Println(e.String()) } // Output: - // https://[e506:816b:407:316:fb4c:ffa0:e208:dc0e]/%F0%97%B0%A0%F0%92%91%9CX/1=%22?%C4%90%F0%90%A9%87%26#%F0%96%AC%B21%CC%88%CC%81D - // http://G.BLoG/%E0%AD%8C~%F0%9F%AF%8A%22%D6%93%E0%A9%AD%E1%B3%930%D0%8A/%C2%BC%E0%B4%BC3%F0%9D%9F%B9%F0%91%91%82%C2%B2%E0%B3%A9%CC%80D/%7C+%F0%9F%82%92+%5D%CC%81%CB%85/%CC%80/%E1%B0%BF/%CD%82K%E0%A5%A9%CC%81#%CC%82 - // https://1.47.4.5:11/+%3E%E2%9F%BCK//A%DB%97%F0%91%99%83$%E0%A0%BD%E2%82%A5%F0%9D%A9%A9%E0%BC%95%E0%B5%A8%3C%E0%BE%AE%F0%97%8A%B1%E2%9E%8E%E0%B9%91$v%CC%80/%CC%94Z%E4%87%94?%F0%96%A9%AEC%C2%B9%E2%8A%A5%F0%92%91%B41%E0%A0%BE%CB%BE%C3%9D%E1%B3%A4%E0%AB%A6%CC%81%CC%86&%E2%A4%88%F0%91%BF%BF%24B%F0%96%BA%90%CC%9A&&%CC%80%C2%A7%E8%93%8B&#%E0%AB%AE%F0%92%91%91 - // http://G.hM/%CC%80%E0%A0%B1%CC%82%CC%80%F0%9E%A5%9F/:%21J%E2%9D%87#L%CC%82%E9%98%8C%22 - // http://1.1.4.6:2/%F0%A7%A8%A4%F0%A1%AD%8D%E2%92%8B0/%DC%B4B?%E2%80%A60%CC%80%C3%B7&%CC%81%CC%A2%21%E0%AF%AB%CC%81%C3%A4&%F0%9F%AF%8A%EA%99%AF%CC%80%E0%A5%AD&%E5%8B%B71&%E1%B7%8F%CC%8B%E2%87%94%E2%90%8E%EA%A3%A0%E0%B5%9A%3D%E5%8E%8A%D9%AAB%F0%A8%83%A2%EF%B8%B4%E0%A0%BD%F0%9D%84%86%C6%81%211A3&%E1%81%8F%23#%CC%80%E0%BF%8B+$ + // https://[e506:816b:407:316:fb4c:ffa0:e208:dc0e]/%25F0%2597%25B0%25A0%25F0%2592%2591%259CX/1=%2522?%C4%90%F0%90%A9%87%26#%F0%96%AC%B21%CC%88%CC%81D + // http://Z.BLOOMBERG:2/%25E2%259C%25A1/1%25F0%2591%2588%25BD/%25F0%259F%25AF%258A%2522%25D6%2593%25E0%25A9%25AD%25E1%25B3%25930%25D0%258A/%25C2%25BC%25E0%25B4%25BC3%25F0%259D%259F%25B9%25F0%2591%2591%2582%25C2%25B2%25E0%25B3%25A9%25CC%2580D/%257C+%25F0%259F%2582%2592+%255D%25CC%2581%25CB%2585/%25CC%2580/%25E1%25B0%25BF/%25CD%2582K%25E0%25A5%25A9%25CC%2581#%CC%82 + // https://1.47.4.5:11/+%253E%25E2%259F%25BCK//A%25DB%2597%25F0%2591%2599%2583$%25E0%25A0%25BD%25E2%2582%25A5%25F0%259D%25A9%25A9%25E0%25BC%2595%25E0%25B5%25A8%253C%25E0%25BE%25AE%25F0%2597%258A%25B1%25E2%259E%258E%25E0%25B9%2591$v%25CC%2580/%25CC%2594Z%25E4%2587%2594?%F0%96%A9%AEC%C2%B9%E2%8A%A5%F0%92%91%B41%E0%A0%BE%CB%BE%C3%9D%E1%B3%A4%E0%AB%A6%CC%81%CC%86&%E2%A4%88%F0%91%BF%BF%24B%F0%96%BA%90%CC%9A&&%CC%80%C2%A7%E8%93%8B&#%E0%AB%AE%F0%92%91%91 + // http://J.HOMESENSE/%25F0%259B%2589%259D%25C2%25B86%25CC%2580%25F0%259E%25A5%259F/:%2521J%25E2%259D%2587#L%CC%82%E9%98%8C%22 + // http://1.1.4.6:2/%25F0%25A7%25A8%25A4%25F0%25A1%25AD%258D%25E2%2592%258B0/%25DC%25B4B?%E2%80%A60%CC%80%C3%B7&%CC%81%CC%A2%21%E0%AF%AB%CC%81%C3%A4&%F0%9F%AF%8A%EA%99%AF%CC%80%E0%A5%AD&%E5%8B%B71&%E1%B7%8F%CC%8B%E2%87%94%E2%90%8E%EA%A3%A0%E0%B5%9A%3D%E5%8E%8A%D9%AAB%F0%A8%83%A2%EF%B8%B4%E0%A0%BD%F0%9D%84%86%C6%81%211A3&%E1%81%8F%23#%CC%80%E0%BF%8B+$ } diff --git a/url_external_test.go b/url_external_test.go index 285f6e3..1aef129 100644 --- a/url_external_test.go +++ b/url_external_test.go @@ -18,7 +18,7 @@ func TestURL(t *testing.T) { t.Parallel() Check(t, func(t *T) { - u := URL().Draw(t, "url").(url.URL) + u := URL().Draw(t, "url") // should be parseable if _, err := url.Parse(u.String()); err != nil { @@ -31,7 +31,7 @@ func TestDomain(t *testing.T) { t.Parallel() Check(t, func(t *T) { - d := Domain().Draw(t, "d").(string) + d := Domain().Draw(t, "d") if got, want := len(d), 255; got > want { t.Errorf("got domain of length %d with maxLenght of %d", got, want) } From b91d28f77141124b9a3dc1b130766e189ba0c7ad Mon Sep 17 00:00:00 2001 From: Walter Scheper Date: Tue, 27 Sep 2022 20:18:50 -0400 Subject: [PATCH 11/12] IPv4 and IPv6 return netip.Addr --- ip_addr.go | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/ip_addr.go b/ip_addr.go index a0f50a3..ebf553d 100644 --- a/ip_addr.go +++ b/ip_addr.go @@ -8,7 +8,7 @@ package rapid import ( "fmt" - "net" + "net/netip" ) const ( @@ -16,11 +16,6 @@ const ( ipv6Len = 16 ) -var ( - ipv4Gen = SliceOfN(Byte(), ipv4Len, ipv4Len) - ipv6Gen = SliceOfN(Byte(), ipv6Len, ipv6Len) -) - type ipGen struct { ipv6 bool } @@ -29,26 +24,26 @@ func (g *ipGen) String() string { return fmt.Sprintf("IP(ipv6=%v)", g.ipv6) } -func (g *ipGen) value(t *T) net.IP { - var gen *Generator[[]byte] +func (g *ipGen) value(t *T) netip.Addr { + var b []byte if g.ipv6 { - gen = ipv6Gen + b = SliceOfN(Byte(), ipv6Len, ipv6Len).Draw(t, g.String()) } else { - gen = ipv4Gen + b = SliceOfN(Byte(), ipv4Len, ipv4Len).Draw(t, g.String()) } - b := gen.Draw(t, g.String()) - return net.IP(b) + addr, _ := netip.AddrFromSlice(b) + return addr } -func IPv4() *Generator[net.IP] { - return newGenerator[net.IP](&ipGen{ +func IPv4() *Generator[netip.Addr] { + return newGenerator[netip.Addr](&ipGen{ ipv6: false, }) } -func IPv6() *Generator[net.IP] { - return newGenerator[net.IP](&ipGen{ +func IPv6() *Generator[netip.Addr] { + return newGenerator[netip.Addr](&ipGen{ ipv6: true, }) } From 8aaba744cdd2d8be723205c14ac3b749916cefa9 Mon Sep 17 00:00:00 2001 From: Walter Scheper Date: Tue, 27 Sep 2022 20:20:41 -0400 Subject: [PATCH 12/12] Use Transform for URL generation This also removes the unnecessary build tag in the example tests. --- url.go | 47 ++++++++++++++++++++++++++------------------ url_example_test.go | 4 ---- url_external_test.go | 6 +++--- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/url.go b/url.go index fa954b0..4be8307 100644 --- a/url.go +++ b/url.go @@ -77,31 +77,40 @@ func (g *urlGenerator) value(t *T) url.URL { case 0: domain = IPv4().Draw(t, "domain").String() } - port := IntRange(0, 2^16-1).Draw(t, "port") - path_ := SliceOf(printableGen).Draw(t, "path") - query := SliceOf(printableGen).Draw(t, "query") - fragment := printableGen.Draw(t, "fragment") - // join domain and port - if port > 0 { - domain += fmt.Sprintf(":%d", port) - } + port := Transform(IntRange(0, 2^16-1), func(i int) string { + if i == 0 { + return "" + } - // URL escape path - for i := range path_ { - path_[i] = url.PathEscape(path_[i]) - } + return fmt.Sprintf(":%d", i) + }).Draw(t, "port") - // url escape query strings - for i := range query { - query[i] = url.QueryEscape(query[i]) - } + path_ := Transform(SliceOf(printableGen), func(paths []string) string { + // URL escape path + for i := range paths { + paths[i] = url.PathEscape(paths[i]) + } + + return strings.Join(paths, "/") + }).Draw(t, "path") + + query := Transform(SliceOf(printableGen), func(queries []string) string { + // url escape query strings + for i := range queries { + queries[i] = url.QueryEscape(queries[i]) + } + + return strings.Join(queries, "&") + }).Draw(t, "query") + + fragment := printableGen.Draw(t, "fragment") return url.URL{ - Host: domain, - Path: strings.Join(path_, "/"), + Host: domain + port, + Path: path_, Scheme: scheme, - RawQuery: strings.Join(query, "&"), + RawQuery: query, Fragment: fragment, } } diff --git a/url_example_test.go b/url_example_test.go index a6ad37c..a6a125a 100644 --- a/url_example_test.go +++ b/url_example_test.go @@ -4,10 +4,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// String generation depends on the Unicode tables, which change with Go versions: -//go:build go1.16 -// +build go1.16 - package rapid_test import ( diff --git a/url_external_test.go b/url_external_test.go index 1aef129..3adecbe 100644 --- a/url_external_test.go +++ b/url_external_test.go @@ -20,9 +20,9 @@ func TestURL(t *testing.T) { Check(t, func(t *T) { u := URL().Draw(t, "url") - // should be parseable + // should be parsable if _, err := url.Parse(u.String()); err != nil { - t.Fatalf("URL returned unparseable url %s: %v", u.String(), err) + t.Fatalf("URL returned unparsable url %s: %v", u.String(), err) } }) } @@ -33,7 +33,7 @@ func TestDomain(t *testing.T) { Check(t, func(t *T) { d := Domain().Draw(t, "d") if got, want := len(d), 255; got > want { - t.Errorf("got domain of length %d with maxLenght of %d", got, want) + t.Errorf("got domain of length %d with maxLength of %d", got, want) } elements := strings.Split(d, ".")