robots.json

[{"status":404,"url":"myflorida.gov","name":"State of Florida","redirect":"https://www.myflorida.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"dc.gov","name":"District of Columbia","redirect":"https://dc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"westutx.gov","name":"City of West University Place, TX","redirect":"https://westutx.gov/robots.txt","sitemap":"https://westutx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cityofwebster.com","name":"City of Webster, TX","redirect":"https://cityofwebster.com/robots.txt","sitemap":"https://cityofwebster.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"wallertexas.gov","name":"City of Waller, TX","redirect":"http://www.wallertexas.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"taylorlakevillage.us","name":"City of Taylor Lake Village, TX","redirect":"http://www.taylorlakevillage.us/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"staffordtx.gov","name":"City of Stafford, TX","redirect":"https://staffordtx.gov/robots.txt","sitemap":"https://staffordtx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"springvalleytx.com","name":"City of Spring Valley Village, TX","redirect":"https://springvalleytx.com/robots.txt","sitemap":"https://springvalleytx.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"southhoustontx.gov","name":"City of South Houston, TX","redirect":"https://southhoustontx.gov/robots.txt","sitemap":"https://southhoustontx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"shoreacrestx.us","name":"City of Shoreacres, TX","redirect":"https://www.shoreacrestx.us/robots.txt","sitemap":"https://www.shoreacrestx.us/wp-sitemap.xml","history":[]},{"status":200,"url":"seabrooktx.gov","name":"City of Seabrook, TX","redirect":"https://seabrooktx.gov/robots.txt","sitemap":"https://seabrooktx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":404,"url":"cityofpineypoint.com","name":"City of Piney Point Village, TX","redirect":"https://www.cityofpineypoint.com/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"nassaubay.com","name":"City of Nassau Bay, TX","redirect":"https://nassaubay.com/robots.txt","sitemap":"https://nassaubay.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"morganspoint-tx.com","name":"City of Morgan’s Point, TX","redirect":"https://www.morganspoint-tx.com/robots.txt","sitemap":"https://www.morganspoint-tx.com/sitemap.xml","history":[]},{"status":200,"url":"cityofkaty.com","name":"City of Katy, TX","redirect":"https://www.cityofkaty.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":404,"url":"jerseyvillagetx.com","name":"City of Jersey Village, TX","redirect":"https://www.jerseyvillagetx.com/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"jacintocity-tx.gov","name":"City of Jacinto City, TX","redirect":"https://www.jacintocity-tx.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cityofhunterscreek.com","name":"City of Hunters Creek Village, TX","redirect":"https://cityofhunterscreek.com/robots.txt","sitemap":"http://cityofhunterscreek.com/sitemap_index.xml","history":[]},{"status":404,"url":"hilshirevillagetexas.com","name":"City of Hilshire Village, TX","redirect":"https://www.hilshirevillagetexas.com/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"thecityofhedwigvillage.com","name":"City of Hedwig Village, TX","redirect":"https://www.thecityofhedwigvillage.com/error/servererror404.htm","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"cityofgalenapark-tx.gov","name":"City of Galena Park, TX","redirect":"https://cityofgalenapark-tx.gov/robots.txt","sitemap":"https://cityofgalenapark-tx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ellago-tx.gov","name":"City of El Lago, TX","redirect":"https://ellago-tx.gov/robots.txt","sitemap":"https://ellago-tx.gov/sitemap_index.xml","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cityofhumbletx.gov","name":"City of Humble, TX","redirect":"https://www.cityofhumbletx.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"bellairetx.gov","name":"City of Bellaire, TX","redirect":"https://bellairetx.gov/robots.txt","sitemap":"https://bellairetx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"waste.gov","name":"Executive Office of the President","redirect":"https://waste.gov/robots.txt","sitemap":null,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"dei.gov","name":"Executive Office of the President","redirect":"https://waste.gov/","sitemap":null,"history":[]},{"status":200,"url":"missouricitytx.gov","name":"City of Missouri, TX","redirect":"https://missouricitytx.gov/robots.txt","sitemap":"https://missouricitytx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":500,"url":"laportetx.gov","name":"City of LaPorte, TX","redirect":"http://www.laportetx.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1743826667324,"status":200,"redirect":"https://laportetx.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://laportetx.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"ci.friendswood.tx.us","name":"City of Friendswood, TX","redirect":"https://ci.friendswood.tx.us/robots.txt","sitemap":"https://ci.friendswood.tx.us/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"deerparktx.gov","name":"City of Deer Park, TX","redirect":"https://deerparktx.gov/robots.txt","sitemap":"https://deerparktx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"baytown.org","name":"City of Baytown, TX","redirect":"https://baytown.org/robots.txt","sitemap":"https://baytown.org/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"harriscountytx.gov","name":"City of Harris County, TX","redirect":"https://www.harriscountytx.gov/robots.txt","sitemap":null,"history":[]},{"status":403,"url":"doge.gov","name":"Department of Government Efficiency","redirect":"https://www.doge.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1743826667324,"status":404,"redirect":"https://doge.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"solanocounty.gov","name":"County of Solano, CA","redirect":"https://solanocounty.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1740965653672,"status":200,"redirect":"https://solanocounty.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1742146243896,"status":500,"redirect":"http://www.solanocounty.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"piedmont.ca.gov","name":"City of Piedmont, CA","redirect":"https://piedmont.ca.gov/robots.txt/Default.aspx","sitemap":null,"valid":true,"allowed":true,"source":false,"history":[]},{"status":200,"url":"eugene-or.gov","name":"City of Eugene, OR","redirect":"https://eugene-or.gov/robots.txt","sitemap":"https://eugene-or.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":500,"url":"cityofsouthfultonga.gov","name":"City of South Fulton, GA","redirect":"http://www.cityofsouthfultonga.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1743826667324,"status":200,"redirect":"https://www.cityofsouthfultonga.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://cityofsouthfultonga.gov/sitemap.xml","source":"robots.txt"}]},{"status":404,"url":"chicago.gov","name":"City of Chicago, IL","redirect":"https://www.chicago.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"yumaaz.gov","name":"City of Yuma, AZ","redirect":"https://www.yumaaz.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"yonkersny.gov","name":"City of Yonkers, NY","redirect":"https://www.yonkersny.gov/robots.txt","sitemap":"https://www.yonkersny.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":500,"url":"worcesterma.gov","name":"City of Worcester, MA","redirect":"https://www.worcesterma.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1743826667324,"status":200,"redirect":"https://www.worcesterma.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"wilmingtonnc.gov","name":"City of Wilmington, NC","redirect":"https://www.wilmingtonnc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"wichita.gov","name":"City of Wichita, KS","redirect":"https://www.wichita.gov/robots.txt","sitemap":"https://wichita.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"wichitafallstx.gov","name":"City of Wichita Falls, TX","redirect":"https://wichitafallstx.gov/robots.txt","sitemap":"https://wichitafallstx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"warrenmi.gov","name":"City of Warren, MI","redirect":"https://warrenmi.gov/robots.txt","sitemap":"https://www.cityofwarren.org/sitemap_index.xml","history":[]},{"status":200,"url":"walkermi.gov","name":"City of Grand Rapids, MI","redirect":"https://walkermi.gov/robots.txt","sitemap":"https://walkermi.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"waco-texas.com","name":"City of Waco, TX","redirect":"https://www.waco-texas.com/robots.txt","sitemap":": https://www.waco-texas.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"visalia.gov","name":"City of Visalia, CA","redirect":"https://www.visalia.city/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"virginiabeach.gov","name":"City of Virginia Beach, VA","redirect":"https://virginiabeach.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"victorvilleca.gov","name":"City of Victorville, CA","redirect":"https://www.victorvilleca.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cityofvancouver.us","name":"City of Vancouver, WA","redirect":"https://www.cityofvancouver.us/robots.txt","sitemap":"https://www.cityofvancouver.us/sitemap_index.xml","history":[]},{"status":200,"url":"vallejo.gov","name":"City of Vallejo, CA","redirect":"https://www.vallejo.gov/robots.txt/Default.aspx","sitemap":null,"valid":true,"allowed":true,"source":false,"history":[]},{"status":400,"url":"tylertexas.gov","name":"City of Tyler, TX","redirect":"http://www.tylertexas.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"tuscaloosa-al.gov","name":"City of Tuscaloosa, AL","redirect":"https://www.tuscaloosa-al.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"tucsonaz.gov","name":"City of Tucson, AZ","redirect":"https://www.tucsonaz.gov/robots.txt","sitemap":": https://www.tucsonaz.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"topeka.org","name":"City of Topeka, KS","redirect":"https://www.topeka.org/robots.txt","sitemap":null,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"toledo.oh.gov","name":"City of Toledo, OH","redirect":"https://toledo.oh.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"toaks.gov","name":"City of Thousand Oaks, CA","redirect":"https://toaks.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"thorntonco.gov","name":"City of Thornton, CO","redirect":"https://www.thorntonco.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"tempe.gov","name":"City of Tempe, AZ","redirect":"https://www.tempe.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"temeculaca.gov","name":"City of Temecula, CA","redirect":"https://temeculaca.gov/robots.txt","sitemap":"https://temeculaca.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"tampa.gov","name":"City of Tampa, FL","redirect":"https://www.tampa.gov/robots.txt","sitemap":"https://www.tampa.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":404,"url":"tacoma.gov","name":"City of Tacoma, WA","redirect":"https://tacoma.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"syr.gov","name":"City of Syracuse, NY","redirect":"https://www.syr.gov/robots.txt","sitemap":": https://www.syr.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"sugarlandtx.gov","name":"City of Sugar Land, TX","redirect":"https://sugarlandtx.gov/robots.txt","sitemap":"https://sugarlandtx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"stpaul.gov","name":"City of Saint Paul, MN","redirect":"https://www.stpaul.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"stocktonca.gov","name":"City of Stockton, CA","redirect":"https://cms3.revize.com/revize/stockton/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"stlouis-mo.gov","name":"City of St. Louis, MO","redirect":"https://www.stlouis-mo.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":404,"url":"stgeorgela.gov","name":"City of Baton Rouge, LA","redirect":"https://stgeorgela.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"sterlingheights.gov","name":"City of Sterling Heights, MI","redirect":"https://sterlingheights.gov/robots.txt","sitemap":"https://sterlingheights.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"stamfordct.gov","name":"City of Stamford, CT","redirect":"https://www.stamfordct.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"springfieldmo.gov","name":"City of Springfield, MO","redirect":"https://www.springfieldmo.gov/robots.txt","sitemap":"https://springfieldmo.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"springfield-ma.gov","name":"City of Springfield, MA","redirect":"https://www.springfield-ma.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"spokanevalleywa.gov","name":"City of Spokane Valley, WA","redirect":"https://spokanevalleywa.gov/robots.txt","sitemap":"https://spokanevalleywa.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":404,"url":"spokane.gov","name":"City of Spokane, WA","redirect":"https://spokane.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"sparksnv.gov","name":"City of Sparks, NV","redirect":"https://cms7files1.revize.com/sparksnv/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"southsideplacetx.gov","name":"City of Houston, TX","redirect":"https://southsideplacetx.gov/robots.txt","sitemap":"https://southsideplacetx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"southbendin.gov","name":"City of South Bend, IN","redirect":"https://southbendin.gov/robots.txt","sitemap":"https://southbendin.gov/sitemap_index.xml","history":[]},{"status":200,"url":"siouxfalls.gov","name":"City of Sioux Falls, SD","redirect":"https://www.siouxfalls.gov/robots.txt","sitemap":": https://www.siouxfalls.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"simivalley.gov","name":"City of Simi Valley, CA","redirect":"https://www.simivalley.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"shreveportla.gov","name":"City of Shreveport, LA","redirect":"https://shreveportla.gov/robots.txt","sitemap":"https://shreveportla.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"sgcityutah.gov","name":"City of St. George, UT","redirect":"https://cms3.revize.com/revize/stgeorge/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"scottsdaleaz.gov","name":"City of Scottsdale, AZ","redirect":"https://www.scottsdaleaz.gov/robots.txt","sitemap":"https://www.scottsdaleaz.gov/sitemap/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1742146243896,"status":404,"redirect":"https://www.scottsdaleaz.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"savannahga.gov","name":"City of Savannah, GA","redirect":"https://savannahga.gov/robots.txt","sitemap":"https://savannahga.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"srcity.org","name":"City of Santa Rosa, CA","redirect":"https://www.srcity.org/robots.txt","sitemap":"https://srcity.org/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"santafenm.gov","name":"City of Santa Fe, NM","redirect":"https://santafenm.gov/robots.txt","sitemap":"https://santafenm.gov/site/sitemap/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"santaclarita.gov","name":"City of Santa Clarita, CA","redirect":"https://santaclarita.gov/robots.txt","sitemap":"https://santaclarita.gov/sitemap_index.xml","history":[]},{"status":200,"url":"santaclaraca.gov","name":"City of Santa Clara, CA","redirect":"https://www.santaclaraca.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"santa-ana.org","name":"City of Santa Ana, CA","redirect":"https://www.santa-ana.org/robots.txt","sitemap":"https://www.santa-ana.org/sitemap_index.xml","history":[]},{"status":200,"url":"sanjoseca.gov","name":"City of San Jose, CA","redirect":"https://www.sanjoseca.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":404,"url":"sandyspringsga.gov","name":"City of Sandy Springs, GA","redirect":"https://www.sandyspringsga.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"sandiego.gov","name":"City of San Diego, CA","redirect":"https://www.sandiego.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"sanbernardino.gov","name":"City of San Bernardino, CA","redirect":"https://www.sanbernardino.gov/robots.txt","sitemap":"https://sanbernardino.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cityofsalinas.org","name":"City of Salinas, CA","redirect":"https://www.cityofsalinas.org/robots.txt","sitemap":": https://www.cityofsalinas.org/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"salemor.gov","name":"City of Salem, OR","redirect":"https://salemor.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"sa.gov","name":"City of San Antonio, TX","redirect":"https://www.sa.gov/robots.txt","sitemap":": https://www.sa.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"rva.gov","name":"City of Richmond, VA","redirect":"https://rva.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"rrnm.gov","name":"City of Rio Rancho, NM","redirect":"https://rrnm.gov/robots.txt","sitemap":"https://rrnm.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"roundrocktexas.gov","name":"City of Round Rock, TX","redirect":"https://www.roundrocktexas.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"rockfordil.gov","name":"City of Rockford, IL","redirect":"https://rockfordil.gov/robots.txt","sitemap":"https://rockfordil.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"rochestermn.gov","name":"City of Rochester, MN","redirect":"https://www.rochestermn.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"richmondca.gov","name":"City of Richmond, CA","redirect":"https://richmondca.gov/robots.txt","sitemap":"https://richmondca.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"rialtoca.gov","name":"City of Rialto, CA","redirect":"https://rialtoca.gov/robots.txt","sitemap":"https://rialtoca.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"rentonwa.gov","name":"City of Renton, WA","redirect":"https://www.rentonwa.gov/robots.txt","sitemap":": https://www.rentonwa.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"reno.gov","name":"City of Reno, NV","redirect":"https://www.reno.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cityofrc.us","name":"City of Rancho Cucamonga, CA","redirect":"https://www.cityofrc.us/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"raleighnc.gov","name":"City of Raleigh, NC","redirect":"https://raleighnc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"quincyma.gov","name":"City of Quincy, MA","redirect":"https://cms7files1.revize.com/quincyma2024/robots.txt","sitemap":null,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"providenceri.gov","name":"City of Providence, RI","redirect":"https://www.providenceri.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"portland.gov","name":"City of Portland, OR","redirect":"https://www.portland.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"pompanobeachfl.gov","name":"City of Pompano Beach, FL","redirect":"https://www.pompanobeachfl.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"pomonaca.gov","name":"City of Pomona, CA","redirect":"https://www.pomonaca.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"plano.gov","name":"City of Plano, TX","redirect":"https://www.plano.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"pittsburghpa.gov","name":"City of Pittsburgh, PA","redirect":"https://www.pittsburghpa.gov/robots.txt","sitemap":": https://www.pittsburghpa.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"peoriagov.org","name":"City of Peoria, IL","redirect":"https://peoriagov.org/robots.txt","sitemap":"https://peoriagov.org/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":403,"url":"peoriaaz.gov","name":"City of Peoria, AZ","redirect":"https://www.peoriaaz.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"pearlandtx.gov","name":"City of Pearland, TX","redirect":"https://www.pearlandtx.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"patersonnj.gov","name":"City of Paterson, NJ","redirect":"https://www.patersonnj.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"pasadenatx.gov","name":"City of Pasadena, TX","redirect":"https://pasadenatx.gov/robots.txt","sitemap":"https://pasadenatx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"pasadena.gov","name":"City of Pasadena, CA","redirect":"https://www.cityofpasadena.net/","sitemap":null,"history":[]},{"status":200,"url":"palmbayflorida.org","name":"City of Palm Bay, FL","redirect":"https://www.palmbayflorida.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"opkansas.org","name":"City of Overland Park, KS","redirect":"https://www.opkansas.org/robots.txt","sitemap":"https://www.opkansas.org/wp-sitemap.xml","history":[]},{"status":200,"url":"orlando.gov","name":"City of Orlando, FL","redirect":"https://www.orlando.gov/robots.txt","sitemap":": https://www.orlando.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cityoforange.org","name":"City of Orange, CA","redirect":"https://www.cityoforange.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"ontarioca.gov","name":"City of Ontario, CA","redirect":"https://www.ontarioca.gov/robots.txt","sitemap":"https://www.ontariothinksbusiness.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"olympiawa.gov","name":"City of Olympia, WA","redirect":"https://www.olympiawa.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"olatheks.gov","name":"City of Olathe, KS","redirect":"https://www.olatheks.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"odessa-tx.gov","name":"City of Odessa, TX","redirect":"https://odessa-tx.gov/robots.txt","sitemap":"https://odessa-tx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"oaklandca.gov","name":"City of Oakland, CA","redirect":"https://www.oaklandca.gov/robots.txt","sitemap":null,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"northcharleston.org","name":"City of North Charleston, SC","redirect":"https://cms2.revize.com/revize/cityofnorthcharleston/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"normanok.gov","name":"City of Norman, OK","redirect":"https://www.normanok.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"norfolk.gov","name":"City of Norfolk, VA","redirect":"https://www.norfolk.gov/robots.txt","sitemap":"https://norfolk.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":500,"url":"nola.gov","name":"City of New Orleans, LA","redirect":"https://nola.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"nnva.gov","name":"City of Newport News, VA","redirect":"https://www.nnva.gov/robots.txt","sitemap":"https://nnva.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"newhavenct.gov","name":"City of New Haven, CT","redirect":"https://www.newhavenct.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"newbraunfels.gov","name":"City of New Braunfels, TX","redirect":"https://newbraunfels.gov/robots.txt","sitemap":"https://newbraunfels.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":403,"url":"newbedford-ma.gov","name":"City of New Bedford, MA","redirect":"https://www.newbedford-ma.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"newarknj.gov","name":"City of newark, NJ","redirect":"https://www.newarknj.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"cityofnampa.us","name":"City of Nampa, ID","redirect":"https://www.cityofnampa.us/robots.txt","sitemap":"https://cityofnampa.us/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"murrietaca.gov","name":"City of Murrieta, CA","redirect":"https://www.murrietaca.gov/robots.txt","sitemap":"https://murrietaca.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"murfreesborotn.gov","name":"City of Murfreesboro, TN","redirect":"https://www.murfreesborotn.gov/robots.txt","sitemap":"https://murfreesborotn.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"moval.gov","name":"City of Moreno Valley, CA","redirect":"https://moval.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"montgomeryal.gov","name":"City of Montgomery, AL","redirect":"https://www.montgomeryal.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"modestogov.com","name":"City of Modesto, CA","redirect":"https://modestogov.com/robots.txt","sitemap":"https://modestogov.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"miramarfl.gov","name":"City of Miramar, FL","redirect":"https://www.miramarfl.gov/robots.txt","sitemap":": https://www.miramarfl.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"miramarfl.gov","name":"City of Miramar, FL","redirect":"https://www.miramarfl.gov/robots.txt","sitemap":": https://www.miramarfl.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"minneapolismn.gov","name":"City of Minneapolis, MN","redirect":"https://www.minneapolismn.gov/robots.txt","sitemap":null,"history":[{"time":1742146243896,"status":404,"redirect":"https://www.minneapolismn.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1743826667324,"status":301,"redirect":"https://www.minneapolismn.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"milwaukee.gov","name":"City of Milwaukee, WI","redirect":"https://www.milwaukee.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"midlandtexas.gov","name":"City of Midland, TX","redirect":"https://midlandtexas.gov/robots.txt","sitemap":"https://midlandtexas.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"miami.gov","name":"City of Miami, FL","redirect":"https://www.miami.gov/robots.txt","sitemap":": https://www.miami.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"miamigardens-fl.gov","name":"City of Miami Gardens, FL","redirect":"https://miamigardens-fl.gov/robots.txt","sitemap":"https://miamigardens-fl.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"mesquitetx.gov","name":"City of Mesquite, TX","redirect":"https://www.cityofmesquite.com/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"memphistn.gov","name":"City of Memphis, TN","redirect":"https://memphistn.gov/robots.txt","sitemap":"https://memphistn.gov/sitemap_index.xml","history":[{"time":1743826667324,"status":200,"redirect":"https://www.memphistn.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.memphistn.gov/sitemap_index.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"manchesternh.gov","name":"City of Manchester, NH","redirect":"https://www.manchesternh.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"cityofmadison.com","name":"City of Madison, WI","redirect":"https://www.cityofmadison.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"lynnma.gov","name":"City of Lynn, MA","redirect":"https://www.lynnma.gov/robots.txt/Default.aspx","sitemap":null,"valid":true,"allowed":true,"source":false,"history":[]},{"status":200,"url":"lowellma.gov","name":"City of Lowell, MA","redirect":"https://lowellma.gov/robots.txt","sitemap":"https://lowellma.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":403,"url":"louisvilleky.gov","name":"City of Louisville, KY","redirect":"https://www.louisvilleky.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"longbeach.gov","name":"City of Long Beach, CA","redirect":"https://longbeach.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"leaguecitytx.gov","name":"City of League City, TX","redirect":"https://leaguecitytx.gov/robots.txt","sitemap":"https://leaguecitytx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"lasvegasnevada.gov","name":"City of Las Vegas, NV","redirect":"https://www.lasvegasnevada.gov/robots.txt","sitemap":"https://www.lasvegasnevada.gov/sitemap.aspx","history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"lascruces.gov","name":"City of Las Cruces, NM","redirect":"https://lascruces.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"cityoflaredo.com","name":"City of Laredo, TX","redirect":"https://www.cityoflaredo.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"lansingmi.gov","name":"City of Lansing, MI","redirect":"https://www.lansingmi.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"lakewood.org","name":"City of Lakewood, CO","redirect":"https://www.lakewood.org/robots.txt","sitemap":": https://www.lakewood.org/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"lacity.gov","name":"City of Los Angeles, CA","redirect":"https://lacity.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"knoxvilletnpolice.gov","name":"City of Knoxville, TN","redirect":"https://knoxvilletnpolice.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"knoxvilletn.gov","name":"City of Knoxville, TN","redirect":"https://www.knoxvilletn.gov/robots.txt/Default.aspx","sitemap":null,"valid":true,"allowed":true,"source":false,"history":[]},{"status":200,"url":"killeentexas.gov","name":"City of Killeen, TX","redirect":"https://killeentexas.gov/robots.txt","sitemap":"https://killeentexas.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"kentwa.gov","name":"City of Kent, WA","redirect":"https://www.kentwa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"kcmo.gov","name":"City of Kansas City, MO","redirect":"https://www.kcmo.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"kannapolisnc.gov","name":"City of Concord, NC","redirect":"https://www.kannapolisnc.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"joliet.gov","name":"City of Joliet, IL","redirect":"https://www.joliet.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"jerseycitynj.gov","name":"City of Jersey City, NJ","redirect":"https://www.jerseycitynj.gov/robots.txt/Default.aspx","sitemap":null,"valid":true,"allowed":true,"source":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"jeffersoncitymo.gov","name":"City of Jefferson City, MO","redirect":"https://jeffersoncitymo.gov/robots.txt","sitemap":null,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"jcmo.gov","name":"City of Jefferson City, MO","redirect":"https://jcmo.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"jacksonville.gov","name":"City of Jacksonville, FL","redirect":"https://www.jacksonville.gov/robots.txt","sitemap":"https://www.jacksonville.gov/cmspages/jaxgov/sitemapxml.aspx","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"jacksonms.gov","name":"City of Jackson, MS","redirect":"https://www.jacksonms.gov/robots.txt","sitemap":"https://www.jacksonms.gov/sitemap_index.xml","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"irvingtx.gov","name":"City of Irving, TX","redirect":"https://irvingtx.gov/robots.txt","sitemap":"https://irvingtx.gov/wp-sitemap.xml","history":[]},{"status":403,"url":"indy.gov","name":"City of Indianapolis, IN","redirect":"http://www.indy.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"indianhill.gov","name":"City of Cincinnati, OH","redirect":"https://indianhill.gov/robots.txt","sitemap":"https://indianhill.gov/sitemap_index.xml","history":[{"time":1743826667324,"status":200,"redirect":"https://indianhill.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://indianhill.gov/wp-sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"independencemo.gov","name":"City of Independence, MO","redirect":"https://www.independencemo.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"huntsvilleal.gov","name":"City of Huntsville, AL","redirect":"http://www.huntsvilleal.gov/robots.txt","sitemap":null,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"huntingtonbeachca.gov","name":"City of Huntington Beach, CA","redirect":"https://cms3.revize.com/revize/huntingtonbeachca/robots.txt","sitemap":null,"history":[]},{"status":500,"url":"houstontx.gov","name":"City of Houston, TX","redirect":"http://www.houstontx.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1743826667324,"status":200,"redirect":"https://houstontx.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"hillsboro-oregon.gov","name":"City of Hillsboro, OR","redirect":"https://www.hillsboro-oregon.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"highpointnc.gov","name":"City of High Point, NC","redirect":"https://www.highpointnc.gov/robots.txt","sitemap":"https://highpointnc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"hialeahfl.gov","name":"City of Hialeah, FL","redirect":"http://www.hialeahfl.gov/robots.txt","sitemap":"http://hialeahfl.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cityofhenderson.com","name":"City of Henderson, NV","redirect":"https://www.cityofhenderson.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"helenamt.gov","name":"City of Helena, MT","redirect":"https://www.helenamt.gov/robots.txt","sitemap":": https://www.helenamt.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":403,"url":"harrisburgpa.gov","name":"City of Harrisburg, PA","redirect":"https://www.harrisburgpa.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"hampton.gov","name":"City of Hampton, VA","redirect":"https://www.hampton.gov/robots.txt","sitemap":"https://hampton.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"greshamoregon.gov","name":"City of Gresham, OR","redirect":"https://www.greshamoregon.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"greensboro-nc.gov","name":"City of Greensboro, NC","redirect":"https://www.greensboro-nc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"greenbaywi.gov","name":"City of Green Bay, WI","redirect":"https://www.greenbaywi.gov/robots.txt","sitemap":"https://greenbaywi.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"greeleygov.com","name":"City of Greeley, CO","redirect":"https://greeleygov.com/robots.txt","sitemap":"http://greeleygov.com/sitemap.gz","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"grandrapidsmi.gov","name":"City of Grand Rapids, MI","redirect":"https://www.grandrapidsmi.gov/robots.txt","sitemap":": https://www.grandrapidsmi.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"goodyearaz.gov","name":"City of Goodyear, AZ","redirect":"https://www.goodyearaz.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"glendaleca.gov","name":"City of Glendale, CA","redirect":"https://www.glendaleca.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":404,"url":"glendaleaz.gov","name":"City of Glendale, AZ","redirect":"https://www.glendaleaz.gov/errors/FileNotFound.aspx","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"garlandtx.gov","name":"City of Garland, TX","redirect":"https://garlandtx.gov/robots.txt","sitemap":"https://garlandtx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"ggcity.org","name":"City of GARDEN GROVE, CA","redirect":"https://ggcity.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"gainesvillefl.gov","name":"City of Gainesville, FL","redirect":"https://www.gainesvillefl.gov/robots.txt","sitemap":": https://www.gainesvillefl.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cityoffullerton.com","name":"City of Fullerton, CA","redirect":"https://www.cityoffullerton.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"friscotexas.gov","name":"City of Frisco, TX","redirect":"https://www.friscotexas.gov/robots.txt","sitemap":"https://friscotexas.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"fresno.gov","name":"City of Fresno, CA","redirect":"http://www.fresno.gov/robots.txt","sitemap":"https://www.fresno.gov/wp-sitemap.xml","history":[]},{"status":200,"url":"fremontpolice.gov","name":"City of Fremont, CA","redirect":"https://www.fremontpolice.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"fremont.gov","name":"City of Fremont, CA","redirect":"https://www.fremont.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"fortworthtexas.gov","name":"City of Fort Worth, TX","redirect":"https://www.fortworthtexas.gov/robots.txt","sitemap":": https://www.fortworthtexas.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"fcgov.com","name":"City of Fort Collins, CO","redirect":"https://www.fcgov.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"fontanaca.gov","name":"City of Fontana, CA","redirect":"https://www.fontanaca.gov/robots.txt","sitemap":"https://fontanaca.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"flpd.gov","name":"City of Fort Lauderdale, FL","redirect":"https://www.flpd.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"fishersin.gov","name":"City of Fishers, IN","redirect":"https://fishersin.gov/robots.txt","sitemap":null,"history":[]},{"status":500,"url":"surpriseaz.gov","name":"City of Surprise, AZ","redirect":"https://www.surpriseaz.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1739509031548,"status":200,"redirect":"https://surpriseaz.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"seattle.gov","name":"City of Seattle, WA","redirect":"https://seattle.gov/robots.txt","sitemap":null,"history":[{"time":1739509031548,"status":200,"redirect":"https://seattle.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1740965653672,"status":200,"redirect":"https://seattle.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"fayettevillenc.gov","name":"City of Fayetteville, NC","redirect":"https://www.fayettevillenc.gov/robots.txt","sitemap":": https://www.fayettevillenc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1739509031548,"status":200,"redirect":"https://www.fayettevillenc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"fayetteville-ar.gov","name":"City of Fayetteville, AR","redirect":"https://www.fayetteville-ar.gov/robots.txt","sitemap":"https://fayetteville-ar.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":404,"url":"fargond.gov","name":"City of Fargo, ND","redirect":"https://fargond.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"everettwa.gov","name":"City of Everett, WA","redirect":"https://www.everettwa.gov/robots.txt","sitemap":"https://everettwa.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"escondido.gov","name":"City of Escondido, CA","redirect":"https://escondido.gov/robots.txt","sitemap":"https://escondido.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"elmonteca.gov","name":"City of El Monte, CA","redirect":"https://elmonteca.gov/robots.txt","sitemap":"https://elmonteca.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":500,"url":"elkgrovecity.org","name":"City of Elk Grove, CA","redirect":"https://elkgrovecity.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1742146243896,"status":200,"redirect":"https://www.elkgrovecity.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":403,"redirect":"https://elkgrove.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"elginil.gov","name":"City of Elgin, IL","redirect":"https://elginil.gov/robots.txt","sitemap":"https://elginil.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"elcajon.gov","name":"City of El Cajon, CA","redirect":"https://www.elcajon.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"eastgrmi.gov","name":"City of Grand Rapids, MI","redirect":"https://eastgrmi.gov/robots.txt","sitemap":"https://eastgrmi.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"durhamnc.gov","name":"City of Durham, NC","redirect":"https://www.durhamnc.gov/robots.txt","sitemap":"https://durhamnc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"detroitmi.gov","name":"City of Detroit, MI","redirect":"https://detroitmi.gov/robots.txt","sitemap":null,"valid":true,"allowed":false,"source":"robots.txt","history":[]},{"status":200,"url":"detroitmi.gov","name":"City of Detroit, MI","redirect":"https://detroitmi.gov/robots.txt","sitemap":null,"valid":true,"allowed":false,"source":"robots.txt","history":[]},{"status":200,"url":"cityofdenton.com","name":"City of Denton, TX","redirect":"https://www.cityofdenton.com/robots.txt","sitemap":"https://cityofdenton.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"dearborn.gov","name":"City of Dearborn, MI","redirect":"https://dearborn.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"daytonohio.gov","name":"City of Dayton, OH","redirect":"https://daytonohio.gov/robots.txt","sitemap":"https://daytonohio.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":500,"url":"dallas.gov","name":"City of Dallas, TX","redirect":"http://www.dallas.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1743826667324,"status":200,"redirect":"https://dallas.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://dallascityhall.com:443/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"cstx.gov","name":"City of College Station, TX","redirect":"https://www.cstx.gov/robots.txt/Default.aspx","sitemap":null,"valid":true,"allowed":true,"source":false,"history":[]},{"status":200,"url":"costamesaca.gov","name":"City of Costa Mesa, CA","redirect":"https://www.costamesaca.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"corpuschristitx.gov","name":"City of Corpus Christi, TX","redirect":"https://www.corpuschristitx.gov/robots.txt","sitemap":"https://www.corpuschristitx.gov/sitemap-xml/","history":[]},{"status":200,"url":"coronaca.gov","name":"City of Corona, CA","redirect":"https://www.coronaca.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cor.net","name":"City of Richardson, TX","redirect":"https://www.cor.net/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"coralsprings.gov","name":"City of CORAL SPRINGS, FL","redirect":"https://www.coralsprings.gov/robots.txt","sitemap":": https://www.coralsprings.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"concordnh.gov","name":"City of Concord, NH","redirect":"https://concordnh.gov/robots.txt","sitemap":"https://concordnh.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"concordnc.gov","name":"City of Concord, NC","redirect":"https://concordnc.gov/robots.txt","sitemap":null,"history":[]},{"status":403,"url":"como.gov","name":"City of Columbia, MO","redirect":"https://www.como.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"columbus.gov","name":"City of Columbus, OH","redirect":"https://www.columbus.gov/robots.txt","sitemap":": https://www.columbus.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"columbiasc.gov","name":"City of Columbia, SC","redirect":"https://columbiasc.gov/robots.txt","sitemap":"https://columbiasc.gov/sitemap_index.xml","history":[]},{"status":200,"url":"coloradosprings.gov","name":"City of Colorado Springs, CO","redirect":"https://coloradosprings.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cityofclovis.com","name":"City of Clovis, CA","redirect":"https://cityofclovis.com/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"clevelandohio.gov","name":"City of Cleveland, OH","redirect":"https://www.clevelandohio.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"clarksvilletn.gov","name":"City of Clarksville, TN","redirect":"https://clarksvilletn.gov/robots.txt","sitemap":"https://clarksvilletn.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":500,"url":"cityofvacaville.gov","name":"City of Vacaville, CA","redirect":"http://www.cityofvacaville.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1743826667324,"status":200,"redirect":"https://www.cityofvacaville.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"cityoftulsa.org","name":"City of Tulsa, OK","redirect":"https://cityoftulsa.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cityofsanmateo.org","name":"City of San Mateo, CA","redirect":"https://cityofsanmateo.org/robots.txt","sitemap":"https://cityofsanmateo.org/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":404,"url":"cityofsacramento.gov","name":"City of Sacramento, CA","redirect":"https://www.cityofsacramento.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"phoenix.gov","name":"City of Phoenix, AZ","redirect":"https://www.phoenix.gov/robots.txt","sitemap":"https://www.phoenix.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1743826667324,"status":200,"redirect":"https://www.phoenix.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"cityofpalmdaleca.gov","name":"City of Palmdale, CA","redirect":"https://cityofpalmdaleca.gov/robots.txt","sitemap":"https://cityofpalmdaleca.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cityofnormandy.gov","name":"City of St. Louis, MO","redirect":"https://www.cityofnormandy.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":403,"url":"cityofmobile.gov","name":"City of Mobile, AL","redirect":"https://www.cityofmobile.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ci.lubbock.tx.us","name":"City of Lubbock, TX","redirect":"http://www.ci.lubbock.tx.us/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1743826667324,"status":200,"redirect":"https://ci.lubbock.tx.us/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"cityoflancasterca.org","name":"City of Lancaster, CA","redirect":"https://www.cityoflancasterca.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cityofirvine.org","name":"City of Irvine, CA","redirect":"https://www.cityofirvine.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"hayward-ca.gov","name":"City of Hayward, CA","redirect":"https://hayward-ca.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"cityofconroe.org","name":"City of Conroe, TX","redirect":"https://cms3.revize.com/revize/conroetx/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"cincinnati-oh.gov","name":"City of Cincinnati, OH","redirect":"https://www.cincinnati-oh.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"chulavistaca.gov","name":"City of Chula Vista, CA","redirect":"https://www.chulavistaca.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"chicoca.gov","name":"City of Chico, CA","redirect":"https://chico.ca.us/Site-Search/?404;https://chico.ca.us:443/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cityofchesapeake.net","name":"City of Chesapeake, VA","redirect":"https://www.cityofchesapeake.net/robots.txt","sitemap":"https://cityofchesapeake.net/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"chattanooga.gov","name":"City of Chattanooga, TN","redirect":"https://chattanooga.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"charlottenc.gov","name":"City of Charlotte, NC","redirect":"https://www.charlottenc.gov/robots.txt","sitemap":": https://www.charlottenc.gov/cmpd/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"charlestonwv.gov","name":"City of Charleston, WV","redirect":"https://charlestonwv.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"charleston-sc.gov","name":"City of Charleston, SC","redirect":"https://charleston-sc.gov/robots.txt","sitemap":"https://charleston-sc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"chandlerazpd.gov","name":"City of Chandler, AZ","redirect":"https://chandlerazpd.gov/robots.txt","sitemap":"https://chandlerazpd.gov/wp-sitemap.xml","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"chandleraz.gov","name":"City of Chandler, AZ","redirect":"https://www.chandleraz.gov/robots.txt","sitemap":"https://www.chandleraz.gov/sitemap.xml","history":[]},{"status":200,"url":"cityofcarrollton.com","name":"City of Carrollton, TX","redirect":"https://www.cityofcarrollton.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":400,"url":"carmelindiana.gov","name":"City of Carmel, IN","redirect":"http://www.carmelindiana.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"carlsbadca.gov","name":"City of Carlsbad, CA","redirect":"https://www.carlsbadca.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"capecoral.gov","name":"City of Cape Coral, FL","redirect":"https://cms4files.revize.com/capecoralfl/robots.txt","sitemap":null,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"capecoral.gov","name":"City of Cape Coral, FL","redirect":"https://cms4files.revize.com/capecoralfl/robots.txt","sitemap":null,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cambridgema.gov","name":"City of Cambridge, MA","redirect":"https://www.cambridgema.gov/robots.txt","sitemap":null,"history":[]},{"status":403,"url":"burbankca.gov","name":"City of Burbank, CA","redirect":"https://www.burbankca.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"bunkerhilltx.gov","name":"City of Houston, TX","redirect":"https://bunkerhilltx.gov/robots.txt","sitemap":"https://bunkerhilltx.gov/xml-sitemap?format=xml","history":[]},{"status":200,"url":"buffalony.gov","name":"City of Buffalo, NY","redirect":"http://www.buffalony.gov/robots.txt","sitemap":"http://buffalony.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"buckeyeaz.gov","name":"City of Buckeye, AZ","redirect":"https://www.buckeyeaz.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"brownsvilletx.gov","name":"City of Brownsville, TX","redirect":"https://www.brownsvilletx.gov/robots.txt","sitemap":"https://brownsvilletx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"brokenarrowok.gov","name":"City of Broken Arrow, OK","redirect":"https://www.brokenarrowok.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"brockton.ma.us","name":"City of Brockton, MA","redirect":"https://brockton.ma.us/robots.txt","sitemap":"https://brockton.ma.us/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"brla.gov","name":"City of Baton Rouge, LA","redirect":"https://www.brla.gov/robots.txt","sitemap":"https://brla.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"bridgeportct.gov","name":"City of Bridgeport, CT","redirect":"https://www.bridgeportct.gov/robots.txt","sitemap":"https://www.bridgeportct.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"bouldercolorado.gov","name":"City of Boulder, CO","redirect":"https://bouldercolorado.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"boston.gov","name":"City of Boston, MA","redirect":"https://www.boston.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":500,"url":"cityofboise.org","name":"City of Boise, ID","redirect":"https://www.cityofboise.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"bismarcknd.gov","name":"City of Bismarck, ND","redirect":"https://bismarcknd.gov/robots.txt","sitemap":"https://bismarcknd.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"birminghamal.gov","name":"City of Birmingham, AL","redirect":"https://www.birminghamal.gov/robots.txt","sitemap":"https://www.birminghamal.gov.ifsight.net/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"billingsmtpublicworks.gov","name":"City of Billings, MT","redirect":"https://billingsmtpublicworks.gov/robots.txt","sitemap":"https://billingsmtpublicworks.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"billingsmt.gov","name":"City of Billings, MT","redirect":"https://billingsmt.gov/robots.txt","sitemap":"https://billingsmt.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"berkeleyca.gov","name":"City of Berkeley, CA","redirect":"https://berkeleyca.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"bendoregon.gov","name":"City of Bend, OR","redirect":"https://www.bendoregon.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"bellevuewa.gov","name":"City of Bellevue, WA","redirect":"https://bellevuewa.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"beaumonttexas.gov","name":"City of Beaumont, TX","redirect":"https://www.beaumonttexas.gov/robots.txt","sitemap":"https://beaumonttexas.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"bakersfieldcity.us","name":"City of Bakersfield, CA","redirect":"https://www.bakersfieldcity.us/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"austintexas.gov","name":"City of Austin, TX","redirect":"https://www.austintexas.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"augustamaine.gov","name":"City of Augusta, ME","redirect":"https://www.augustamaine.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"augustaga.gov","name":"City of Augusta, GA","redirect":"https://www.augustaga.gov/robots.txt","sitemap":"https://augustaga.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":500,"url":"atlantaga.gov","name":"City of Atlanta, GA","redirect":"http://www.atlantaga.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1743826667324,"status":200,"redirect":"https://www.atlantaga.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"arvadaco.gov","name":"City of Arvada, CO","redirect":"https://www.arvadaco.gov/robots.txt","sitemap":"https://arvadaco.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"arlingtontx.gov","name":"City of Arlington, TX","redirect":"https://www.arlingtontx.gov/robots.txt","sitemap":": https://www.arlingtontx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1742146243896,"status":200,"redirect":"https://www.arlingtontx.gov/robots.txt/Default.aspx","valid":true,"allowed":true,"sitemap":null,"source":false}]},{"status":200,"url":"antiochca.gov","name":"City of Antioch, CA","redirect":"https://www.antiochca.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"annapolis.gov","name":"City of Annapolis, MD","redirect":"https://www.annapolis.gov/robots.txt","sitemap":"https://annapolis.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"anaheim.net","name":"City of Anaheim, CA","redirect":"https://anaheim.net/robots.txt","sitemap":"https://anaheim.net/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"amarillo.gov","name":"City of Amarillo, TX","redirect":"https://www.amarillo.gov/robots.txt","sitemap":null,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"allentx.gov","name":"City of Allen, TX","redirect":"https://cms3.revize.com/revize/allentx/robots.txt","sitemap":null,"history":[{"time":1743826667324,"status":404,"redirect":"https://allentx.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"allentownpa.gov","name":"City of Allentown, PA","redirect":"https://www.allentownpa.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"alexandriava.gov","name":"City of Alexandria, VA","redirect":"https://alexandriava.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cabq.gov","name":"City of Albuquerque, NM","redirect":"https://www.cabq.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"albanyny.gov","name":"City of Albany, NY","redirect":"https://albanyny.gov/robots.txt","sitemap":"https://albanyny.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"akronohio.gov","name":"City of Akron, OH","redirect":"https://cms2.revize.com/revize/akronoh/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"abilenetx.gov","name":"City of Abilene, TX","redirect":"https://abilenetx.gov/robots.txt","sitemap":"https://abilenetx.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"apps.nationalmap.gov","name":"United States Geological Survey","redirect":"https://apps.nationalmap.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":500,"url":"ipdaac.usgs.gov","name":"United States Geological Survey","redirect":"http://www.ipdaac.usgs.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"water.usgs.gov","name":"United States Geological Survey","redirect":"https://water.usgs.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":404,"url":"volcanoes.usgs.gov","name":"United States Geological Survey","redirect":"https://volcanoes.usgs.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"ers.cr.usgs.gov","name":"United States Geological Survey","redirect":"https://ers.cr.usgs.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"ngmdb.usgs.gov","name":"United States Geological Survey","redirect":"https://ngmdb.usgs.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"earthexplorer.usgs.gov","name":"United States Geological Survey","redirect":"https://earthexplorer.usgs.gov/robots.txt","sitemap":null,"history":[]},{"status":200,"url":"store.usgs.gov","name":"United States Geological Survey","redirect":"https://store.usgs.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":404,"url":"dashboard.waterdata.usgs.gov","name":"United States Geological Survey","redirect":"https://dashboard.waterdata.usgs.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"waterdata.usgs.gov","name":"United States Geological Survey","redirect":"https://waterdata.usgs.gov/robots.txt","sitemap":"http://waterdata.usgs.gov/sitemapWaterdata.xml","valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"earthquake.usgs.gov","name":"United States Geological Survey","redirect":"https://www.usgs.gov/programs/earthquake-hazards","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[]},{"status":200,"url":"bha.colorado.gov","name":"Colorado Behavioral Health Administration","redirect":"https://bha.colorado.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"cityofmonongahela-pa.gov","name":"City of Monongahela, Pennsylvania","redirect":"https://www.cityofmonongahela-pa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"altoonawater.gov","name":"Altoona Water Authority","redirect":"https://www.altoonawater.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[]},{"status":200,"url":"jocogov.org","name":"Johnson County, Kansas","redirect":"https://www.jocogov.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.jocogov.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"usarmyband.com","name":"Department of Defense","redirect":"https://usarmyband.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://usarmyband.com/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"vavmc.com","name":"Department of Veterans Affairs","redirect":"https://vavmc.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.vavmc.com","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"http://www.vavmc.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1737430190371,"status":500,"redirect":"http://www.vavmc.com/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"vaitcampus.com","name":"Department of Veterans Affairs","redirect":"https://www.vaitcampus.com/robots.txt","sitemap":"https://vaitcampus.com/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://vaitcampus.com/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://vaitcampus.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1737430190371,"status":500,"redirect":"http://www.vaitcampus.com/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"http://www.vaitcampus.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"treatmentworksforvets.org","name":"Department of Veterans Affairs","redirect":"https://treatmentworksforvets.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://treatmentworksforvets.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"shopvcs.com","name":"Department of Veterans Affairs","redirect":"https://www.vacanteen.va.gov/VACANTEEN/specialorder.asp","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://shopvcs.va.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://shopvcs.va.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"otianywhere.net","name":"United States Agency for International Development","redirect":"https://otianywhere.net/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://otianywhere.net/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"monitor.net.co","name":"United States Agency for International Development","redirect":"https://www.monitor.net.co/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://monitor.net.co/intro.aspx","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://monitor.net.co/intro.aspx","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://monitor.net.co/Intro.aspx?ReturnUrl=%2frobots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"https://monitor.net.co/intro.aspx","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"smithsonianstudentadventures.com","name":"Smithsonian Institution","redirect":"https://smithsonianstudenttravel.org/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://smithsonianstudenttravel.org/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://smithsonianstudenttravel.org/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.smithsonianstudentadventures.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"smithsonianearthtv.com","name":"Smithsonian Institution","redirect":"http://www.smithsonianearthtv.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"smithsoniancraftshow.org","name":"Smithsonian Institution","redirect":"https://smithsoniancraftshow.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://smithsoniancraftshow.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"smithsoniancraft2wear.org","name":"Smithsonian Institution","redirect":"https://smithsoniancraftshow.org/craft2wear","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://smithsoniancraftshow.org/craft2wear","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://smithsoniancraftshow.org/craft2wear","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.smithsoniancraft2wear.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"smithsonianchambermusic.org","name":"Smithsonian Institution","redirect":"https://smithsonianchambermusic.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://smithsonianchambermusic.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"smithsonianapa.org","name":"Smithsonian Institution","redirect":"http://www.smithsonianapa.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://apa.si.edu/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"https://apa.si.edu/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"seecstories.com","name":"Smithsonian Institution","redirect":"https://seecstories.com/robots.txt","sitemap":"https://seecstories.com/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://seecstories.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://seecstories.com/sitemap.xml"}]},{"status":200,"url":"searchablemuseum.com","name":"Smithsonian Institution","redirect":"https://www.searchablemuseum.com/robots.txt","sitemap":"https://www.searchablemuseum.com/sitemap-0.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.searchablemuseum.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.searchablemuseum.com/sitemap-0.xml"}]},{"status":200,"url":"paleoglot.org","name":"Smithsonian Institution","redirect":"https://naturalhistory.si.edu/research/paleobiology/research/carrano-lab/polyglot-paleontologist","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://naturalhistory.si.edu/research/paleobiology/research/carrano-lab/polyglot-paleontologist","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://naturalhistory.si.edu/research/paleobiology/research/carrano-lab/polyglot-paleontologist","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://paleoglot.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":403,"url":"nmaahcstore.com","name":"Smithsonian Institution","redirect":"https://www.nmaahcstore.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"museumonmainstreet.org","name":"Smithsonian Institution","redirect":"https://museumonmainstreet.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://museumonmainstreet.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"mughalgardens.org","name":"Smithsonian Institution","redirect":"https://mughalgardens.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://mughalgardens.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ellajenkins.com","name":"Smithsonian Institution","redirect":"https://ellajenkins.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://ellajenkins.com/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ecosystemsontheedge.org","name":"Smithsonian Institution","redirect":"https://ecosystemsontheedge.org/robots.txt","sitemap":"https://ecosystemsontheedge.org/wp-sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://ecosystemsontheedge.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://ecosystemsontheedge.org/wp-sitemap.xml"}]},{"status":200,"url":"discoverytheater.org","name":"Smithsonian Institution","redirect":"https://discoverytheater.org/robots.txt","sitemap":"https://discoverytheater.org/sitemap.xml.gz","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":301,"redirect":"http://discoverytheater.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"cicadamerch.com","name":"Smithsonian Institution","redirect":"https://www.cicadamerch.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"artcollectorsprogram.org","name":"Smithsonian Institution","redirect":"https://artcollectorsprogram.org/robots.txt","sitemap":"https://artcollectorsprogram.org/sitemap.xml.gz","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://artcollectorsprogram.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://artcollectorsprogram.org/sitemap.xml.gz"}]},{"status":500,"url":"howeycoins.com","name":"Securities and Exchange Commission","redirect":"http://www.howeycoins.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"pbgc.us","name":"Pension Benefit Guaranty Corporation","redirect":"https://www.pbgc.gov/robots.txt","sitemap":"https://www.pbgc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pbgc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.pbgc.gov/sitemap.xml"}]},{"status":200,"url":"pbgc.net","name":"Pension Benefit Guaranty Corporation","redirect":"https://www.pbgc.gov/robots.txt","sitemap":"https://www.pbgc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pbgc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.pbgc.gov/sitemap.xml"}]},{"status":200,"url":"pbgc.com","name":"Pension Benefit Guaranty Corporation","redirect":"https://www.pbgc.gov/robots.txt","sitemap":"https://www.pbgc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pbgc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.pbgc.gov/sitemap.xml"}]},{"status":200,"url":"pbgc.biz","name":"Pension Benefit Guaranty Corporation","redirect":"https://www.pbgc.gov/robots.txt","sitemap":"https://www.pbgc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pbgc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.pbgc.gov/sitemap.xml"}]},{"status":404,"url":"xsp.org","name":"Office of Personnel Management","redirect":"http://www.xsp.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://xsp.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"usajobsrecruit.us","name":"Office of Personnel Management","redirect":"http://www.usajobsrecruit.us/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"http://usajobsrecruit.us/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.usajobsrecruit.us","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"http://www.usajobsrecruit.us/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":500,"redirect":"http://www.usajobsrecruit.us/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"usajobsrecruit.com","name":"Office of Personnel Management","redirect":"http://www.usajobsrecruit.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"http://usajobsrecruit.com/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://usajobsrecruit.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"http://www.usajobsrecruit.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":500,"redirect":"http://www.usajobsrecruit.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":429,"url":"usajobsassess.us","name":"Office of Personnel Management","redirect":"http://www.usajobsassess.us/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":429,"redirect":"http://usajobsassess.us/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1729037740707,"status":500,"redirect":"http://www.usajobsassess.us/","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":429,"redirect":"http://www.usajobsassess.us/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":429,"redirect":"http://www.usajobsassess.us/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1742146243896,"status":429,"redirect":"http://www.usajobsassess.us/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":429,"redirect":"http://www.usajobsassess.us/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"usajobsassess.com","name":"Office of Personnel Management","redirect":"http://www.usajobsassess.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"http://usajobsassess.com/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.usajobsassess.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"http://www.usajobsassess.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":500,"redirect":"http://www.usajobsassess.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"usajobs.com","name":"Office of Personnel Management","redirect":"https://www.usajobs.gov/robots.txt","sitemap":"https://www.usajobs.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usajobs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usajobs.gov/sitemap.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"federalcampaign.org","name":"Office of Personnel Management","redirect":"http://www.federalcampaign.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"http://federalcampaign.org/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.federalcampaign.org","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"http://www.federalcampaign.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":500,"redirect":"http://www.federalcampaign.org/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"combinedfederal.org","name":"Office of Personnel Management","redirect":"http://www.combinedfederal.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"http://combinedfederal.org/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"http://www.combinedfederal.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":500,"redirect":"http://www.combinedfederal.org/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cfcgiving.org","name":"Office of Personnel Management","redirect":"http://www.cfcgiving.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"http://cfcgiving.org/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"http://www.cfcgiving.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":500,"redirect":"http://www.cfcgiving.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fedroom.com","name":"General Services Administration","redirect":"https://fedroom.com/robots.txt","sitemap":null,"valid":true,"allowed":false,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://fedroom.com/robots.txt","valid":true,"allowed":false,"sitemap":null}]},{"status":500,"url":"doi.net","name":"Department of the Interior","redirect":"http://www.doi.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"rfj.tips","name":"Department of State","redirect":"https://rfj.tips/404","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://rfj.tips/404","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"workforcegps.org","name":"Department of Labor","redirect":"http://www.workforcegps.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"whatcanyoudocampaign.org","name":"Department of Labor","redirect":"https://www.whatcanyoudocampaign.org/robots.txt","sitemap":"https://www.whatcanyoudocampaign.org/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.whatcanyoudocampaign.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.whatcanyoudocampaign.org/sitemap_index.xml"}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"ustedquepuedehacer.org","name":"Department of Labor","redirect":"https://www.whatcanyoudocampaign.org/espanol/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whatcanyoudocampaign.org/espanol/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whatcanyoudocampaign.org/espanol/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":302,"redirect":"http://www.ustedquepuedehacer.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"useuosh.org","name":"Department of Labor","redirect":"https://www.useuosh.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.useuosh.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"servicelocator.org","name":"Department of Labor","redirect":"https://www.servicelocator.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.servicelocator.org/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1734148194754,"status":403,"redirect":"https://www.careeronestop.org/localhelp/service-locator.aspx","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"peatworks.org","name":"Department of Labor","redirect":"https://www.dol.gov/agencies/odep/program-areas/employment-supports/technology/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.peatworks.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.peatworks.org/sitemap_index.xml"},{"time":1743826667324,"status":200,"redirect":"https://www.peatworks.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.peatworks.org/sitemap_index.xml","source":"robots.txt"}]},{"status":200,"url":"onetcenter.org","name":"Department of Labor","redirect":"https://www.onetcenter.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.onetcenter.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"neapmap.com","name":"Department of Labor","redirect":"https://neapmap.com/robots.txt","sitemap":"https://neapmap.com/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"http://neapmap.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://neapmap.com/sitemap_index.xml"}]},{"status":200,"url":"mynextmove.org","name":"Department of Labor","redirect":"https://www.mynextmove.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.mynextmove.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"joinjobcorps.com","name":"Department of Labor","redirect":"http://www.joinjobcorps.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"jobcorps.org","name":"Department of Labor","redirect":"https://jobcorps.gov//","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"jctalentworks.com","name":"Department of Labor","redirect":"http://www.jctalentworks.com/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1734148194754,"status":404,"redirect":"https://info.joinjobcorps.com/searchcareersrobots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":302,"redirect":"http://www.jctalentworks.com/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":404,"redirect":"https://info.joinjobcorps.com/searchcareersrobots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":302,"redirect":"http://www.jctalentworks.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":401,"url":"jcsuccess.org","name":"Department of Labor","redirect":"https://www.jcsuccess.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":401,"redirect":"https://jcsuccess.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"http://www.jcsuccess.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"jcmarketplace.com","name":"Department of Labor","redirect":"https://jcmarketplace.com/robots.txt","sitemap":"https://jcmarketplace.com/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"http://jcmarketplace.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://jcmarketplace.com/sitemap_index.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"inclusiveapprenticeship.org","name":"Department of Labor","redirect":"https://inclusiveapprenticeship.org/robots.txt","sitemap":"https://inclusiveapprenticeship.org/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.dol.gov/agencies/odep/program-areas/apprenticeship/pia","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dol.gov/agencies/odep/program-areas/apprenticeship/pia","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://inclusiveapprenticeship.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":200,"redirect":"https://www.dol.gov/agencies/odep/program-areas/apprenticeship/pia","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"https://inclusiveapprenticeship.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"euusosh.org","name":"Department of Labor","redirect":"https://www.euusosh.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.euusosh.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"epolicyworks.org","name":"Department of Labor","redirect":"https://epolicyworks.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"http://epolicyworks.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":404,"redirect":"http://www.epolicyworks.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"capeyouth.org","name":"Department of Labor","redirect":"https://capeyouth.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://capeyouth.org/robots.txt","valid":true,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://capeyouth.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":410,"redirect":"https://www.capeyouth.org/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"bonds4jobs.com","name":"Department of Labor","redirect":"https://bonds4jobs.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://bonds4jobs.com/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":202,"url":"alx.org","name":"Department of Labor","redirect":"https://www.careeronestop.org/Credentials/default.aspx","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":403,"redirect":"https://www.careeronestop.org/Credentials/default.aspx","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.alx.org/","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":403,"redirect":"https://www.careeronestop.org/Credentials/default.aspx","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":202,"redirect":"https://www.careeronestop.org/Credentials/default.aspx","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.alx.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":202,"url":"acinet.org","name":"Department of Labor","redirect":"https://www.careeronestop.org/Toolkit/ACINet.aspx","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":403,"redirect":"https://www.careeronestop.org/Toolkit/ACINet.aspx","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.acinet.org/","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":403,"redirect":"https://www.careeronestop.org/Toolkit/ACINet.aspx","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":202,"redirect":"https://www.careeronestop.org/Toolkit/ACINet.aspx","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.acinet.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"protectivedns.net","name":"Department of Homeland Security","redirect":"https://www.protectivedns.cisa.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"cyberstorm8.com","name":"Department of Homeland Security","redirect":"http://www.cyberstorm8.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cdmdashboard.com","name":"Department of Homeland Security","redirect":"http://www.cdmdashboard.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"worklife4you.com","name":"Department of Health and Human Services","redirect":"https://www.worklife4you.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.worklife4you.com/index.html","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"vetmedinfo.net","name":"Department of Health and Human Services","redirect":"http://www.vetmedinfo.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"vetmed.wiki","name":"Department of Health and Human Services","redirect":"http://www.vetmed.wiki/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"vaccinefinder.org","name":"Department of Health and Human Services","redirect":"http://www.vaccinefinder.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.vaccines.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.vaccines.gov/en/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.vaccines.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"https://www.vaccines.gov/en/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"uspreventiveservicestaskforce.org","name":"Department of Health and Human Services","redirect":"https://www.uspreventiveservicestaskforce.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.uspreventiveservicestaskforce.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"thesisterstudy.org","name":"Department of Health and Human Services","redirect":"http://www.thesisterstudy.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://sisterstudy.niehs.nih.gov/English/index1.htm","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"thehearttruth.org","name":"Department of Health and Human Services","redirect":"https://nhlbi.nih.gov/education/heart-truth/CHW/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.nhlbi.nih.gov/education/heart-truth/CHW","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://thehearttruth.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":200,"redirect":"https://www.nhlbi.nih.gov/education/heart-truth/CHW","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"thehearttruth.net","name":"Department of Health and Human Services","redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://thehearttruth.net/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":200,"redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":404,"redirect":"http://www.thehearttruth.net/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"thehearttruth.com","name":"Department of Health and Human Services","redirect":"https://nhlbi.nih.gov/education/heart-truth/CHW/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.nhlbi.nih.gov/education/heart-truth/CHW","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://thehearttruth.com/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":200,"redirect":"https://www.nhlbi.nih.gov/education/heart-truth/CHW","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"ssbgportal.net","name":"Department of Health and Human Services","redirect":"https://ssbgportal.acf.hhs.gov/ErrorPage","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://ssbgportal.acf.hhs.gov/ErrorPage","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"sitenv.org","name":"Department of Health and Human Services","redirect":"http://www.sitenv.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"sisterstudy.org","name":"Department of Health and Human Services","redirect":"https://sisterstudy.niehs.nih.gov/English/index1.htm","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://sisterstudy.niehs.nih.gov/English/index1.htm","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://sisterstudy.niehs.nih.gov/English/index1.htm","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.sisterstudy.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"scienceforum.sc","name":"Department of Health and Human Services","redirect":"http://www.scienceforum.sc/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"racwarehouse.org","name":"Department of Health and Human Services","redirect":"https://www.racwarehouse.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.racwarehouse.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"quehacerahora.org","name":"Department of Health and Human Services","redirect":"https://www.cdc.gov/howrightnow/espanol/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cdc.gov/howrightnow/espanol/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"https://www.quehacerahora.org/","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":301,"redirect":"https://quehacerahora.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":401,"redirect":"https://www.quehacerahora.org/","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"https://www.cdc.gov/howrightnow/espanol/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":401,"redirect":"https://www.quehacerahora.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"qualitynet.org","name":"Department of Health and Human Services","redirect":"https://qualitynet.cms.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://qualitynet.cms.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"pubmedcentral.us","name":"Department of Health and Human Services","redirect":"http://www.pubmedcentral.us/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pubmedcentral.info","name":"Department of Health and Human Services","redirect":"http://www.pubmedcentral.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pubmed-central.org","name":"Department of Health and Human Services","redirect":"http://www.pubmed-central.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pubmed-central.net","name":"Department of Health and Human Services","redirect":"http://www.pubmed-central.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pubmed-central.info","name":"Department of Health and Human Services","redirect":"http://www.pubmed-central.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pubmed-central.com","name":"Department of Health and Human Services","redirect":"http://www.pubmed-central.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pubchem.us","name":"Department of Health and Human Services","redirect":"http://www.pubchem.us/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pubchem.info","name":"Department of Health and Human Services","redirect":"http://www.pubchem.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pubchem.biz","name":"Department of Health and Human Services","redirect":"http://www.pubchem.biz/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"psoppc.info","name":"Department of Health and Human Services","redirect":"http://www.psoppc.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"proteincapture.org","name":"Department of Health and Human Services","redirect":"http://www.proteincapture.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"presidentschallenge.org","name":"Department of Health and Human Services","redirect":"http://presidentschallenge.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://presidentschallenge.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"phs-nurse.org","name":"Department of Health and Human Services","redirect":"https://www.phs-nurse.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://phs-nurse.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"http://www.phs-nurse.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":404,"redirect":"https://www.phs-nurse.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":false}]},{"status":500,"url":"phiresearchlab.org","name":"Department of Health and Human Services","redirect":"http://www.phiresearchlab.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pcorcds-ln.org","name":"Department of Health and Human Services","redirect":"http://www.pcorcds-ln.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pccqualitymonitoring.com","name":"Department of Health and Human Services","redirect":"http://www.pccqualitymonitoring.com/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1734148194754,"status":500,"redirect":"http://www.pccqualitymonitoring.com/","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"https://pccqualitymonitoring.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"pccds-ln.org","name":"Department of Health and Human Services","redirect":"https://pccds-ln.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.pccds-ln.org","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"oahpmdata.net","name":"Department of Health and Human Services","redirect":"https://oahpmdata.net/ErrorPage","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.oahpmdata.net","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"nlmgiftshop.net","name":"Department of Health and Human Services","redirect":"http://www.nlmgiftshop.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nlmgiftshop.info","name":"Department of Health and Human Services","redirect":"http://www.nlmgiftshop.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nlmgiftshop.com","name":"Department of Health and Human Services","redirect":"http://www.nlmgiftshop.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nlm.org","name":"Department of Health and Human Services","redirect":"http://www.nlm.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nlm.net","name":"Department of Health and Human Services","redirect":"http://www.nlm.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nihseniorhealth.org","name":"Department of Health and Human Services","redirect":"http://www.nihseniorhealth.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nihseniorhealth.net","name":"Department of Health and Human Services","redirect":"http://www.nihseniorhealth.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nihseniorhealth.info","name":"Department of Health and Human Services","redirect":"http://www.nihseniorhealth.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nihseniorhealth.com","name":"Department of Health and Human Services","redirect":"http://www.nihseniorhealth.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nihseniorhealth.biz","name":"Department of Health and Human Services","redirect":"http://www.nihseniorhealth.biz/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"nih.com","name":"Department of Health and Human Services","redirect":"https://www.nih.gov/","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.nih.gov/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":200,"redirect":"https://www.nih.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.nih.gov/sitemap.xml","source":"robots.txt"},{"time":1740965653672,"status":200,"redirect":"https://www.nih.gov/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1742146243896,"status":200,"redirect":"https://www.nih.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.nih.gov/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"nhlbiwgs.org","name":"Department of Health and Human Services","redirect":"https://topmed.nhlbi.nih.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://topmed.nhlbi.nih.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"https://topmed.nhlbi.nih.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://nhlbiwgs.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"ncihub.org","name":"Department of Health and Human Services","redirect":"http://www.ncihub.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":401,"url":"nci-fyi.org","name":"Department of Health and Human Services","redirect":"https://nci-fyi.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":401,"redirect":"https://nci-fyi.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ncats.io","name":"Department of Health and Human Services","redirect":"https://ncats.nih.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://ncats.nih.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://ncats.nih.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://ncats.io/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"ncands.net","name":"Department of Health and Human Services","redirect":"https://ncands.acf.hhs.gov/Account/Login?ReturnUrl=%2F","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://ncands.acf.hhs.gov/Account/Login?ReturnUrl=%2F","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://ncands.acf.hhs.gov/Account/Login?ReturnUrl=%2F","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://ncands.acf.hhs.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":200,"redirect":"https://ncands.acf.hhs.gov/Account/Login?ReturnUrl=%2F","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":500,"redirect":"https://ncands.acf.hhs.gov/Account/Login?ReturnUrl=%2F","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://ncands.acf.hhs.gov/Account/Login?ReturnUrl=%2F","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":500,"redirect":"https://ncands.acf.hhs.gov/Account/Login?ReturnUrl=%2F","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"namrs.net","name":"Department of Health and Human Services","redirect":"https://namrs.net/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://namrs.net/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://namrs.net/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":500,"redirect":"https://www.namrs.net/Account/Login?ReturnUrl=%2F","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"mesh.org","name":"Department of Health and Human Services","redirect":"http://www.mesh.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"mepstech.com","name":"Department of Health and Human Services","redirect":"https://nojs-rotate.domaincntrol.com/?d=nojs&t=3","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"http://mepstech.com/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://mepstech.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":404,"url":"mepsdocs.org","name":"Department of Health and Human Services","redirect":"https://mepsdocs.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://mepsdocs.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"https://www.mepsdocs.org/","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":404,"redirect":"https://mepsdocs.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":500,"redirect":"http://www.mepsdocs.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"medlneplus.info","name":"Department of Health and Human Services","redirect":"http://www.medlneplus.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"medlneplus.com","name":"Department of Health and Human Services","redirect":"http://www.medlneplus.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"medlineplus.org","name":"Department of Health and Human Services","redirect":"http://www.medlineplus.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"medlineplus.net","name":"Department of Health and Human Services","redirect":"http://www.medlineplus.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"medlineplus.com","name":"Department of Health and Human Services","redirect":"http://www.medlineplus.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"medlars.us","name":"Department of Health and Human Services","redirect":"http://www.medlars.us/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"medlars.org","name":"Department of Health and Human Services","redirect":"http://www.medlars.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"medlars.net","name":"Department of Health and Human Services","redirect":"http://www.medlars.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"medlars.info","name":"Department of Health and Human Services","redirect":"http://www.medlars.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"medlars.com","name":"Department of Health and Human Services","redirect":"http://www.medlars.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"medicalbillrights.org","name":"Department of Health and Human Services","redirect":"http://www.medicalbillrights.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://medicalbillrights.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.medicalbillrights.org","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"medicalbillrights.net","name":"Department of Health and Human Services","redirect":"https://medicalbillrights.net/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://medicalbillrights.net/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.medicalbillrights.net","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"medicalbillrights.com","name":"Department of Health and Human Services","redirect":"https://medicalbillrights.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://medicalbillrights.com/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.medicalbillrights.com","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"loansomedoc.org","name":"Department of Health and Human Services","redirect":"http://www.loansomedoc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"loansomedoc.net","name":"Department of Health and Human Services","redirect":"http://www.loansomedoc.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"informationrx.net","name":"Department of Health and Human Services","redirect":"http://www.informationrx.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"informationrx.info","name":"Department of Health and Human Services","redirect":"http://www.informationrx.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"igm.org","name":"Department of Health and Human Services","redirect":"http://www.igm.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"howrightnow.org","name":"Department of Health and Human Services","redirect":"https://www.cdc.gov/howrightnow/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cdc.gov/howrightnow/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.howrightnow.org","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":301,"redirect":"https://www.howrightnow.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1734148194754,"status":401,"redirect":"https://www.howrightnow.org/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.cdc.gov/howrightnow/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1740965653672,"status":500,"redirect":"https://www.howrightnow.org/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"howrightnow.online","name":"Department of Health and Human Services","redirect":"https://www.cdc.gov/howrightnow/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cdc.gov/howrightnow/","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":500,"redirect":"https://www.howrightnow.online/","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":401,"redirect":"https://www.howrightnow.online/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.cdc.gov/howrightnow/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1740965653672,"status":401,"redirect":"https://www.howrightnow.online/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"howrightnow.com","name":"Department of Health and Human Services","redirect":"https://www.cdc.gov/howrightnow/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cdc.gov/howrightnow/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.howrightnow.com","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":301,"redirect":"https://howrightnow.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":401,"redirect":"https://www.howrightnow.com/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.cdc.gov/howrightnow/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1740965653672,"status":500,"redirect":"https://www.howrightnow.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"hfpp-ttp.org","name":"Department of Health and Human Services","redirect":"http://www.hfpp-ttp.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"hearttruth.org","name":"Department of Health and Human Services","redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.hearttruth.org","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://hearttruth.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"hearttruth.net","name":"Department of Health and Human Services","redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.hearttruth.net","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://hearttruth.net/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"hearttruth.com","name":"Department of Health and Human Services","redirect":"https://nhlbi.nih.gov/education/heart-truth/CHW/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.hearttruth.com","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://hearttruth.com/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":200,"redirect":"https://www.nhlbi.nih.gov/education/heart-truth/CHW","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"hcgov.us","name":"Department of Health and Human Services","redirect":"https://hcgov.us/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://hcgov.us/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"hapmap.org","name":"Department of Health and Human Services","redirect":"http://www.hapmap.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"gratefulmed.org","name":"Department of Health and Human Services","redirect":"http://www.gratefulmed.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"gratefulmed.net","name":"Department of Health and Human Services","redirect":"http://www.gratefulmed.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"gratefulmed.info","name":"Department of Health and Human Services","redirect":"http://www.gratefulmed.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"gratefulmed.com","name":"Department of Health and Human Services","redirect":"http://www.gratefulmed.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"genbank.xyz","name":"Department of Health and Human Services","redirect":"http://www.genbank.xyz/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"genbank.org","name":"Department of Health and Human Services","redirect":"http://www.genbank.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"genbank.net","name":"Department of Health and Human Services","redirect":"http://www.genbank.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"genbank.com","name":"Department of Health and Human Services","redirect":"http://www.genbank.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"gem-measures.org","name":"Department of Health and Human Services","redirect":"https://cancercontrol.cancer.gov/brp/research/group-evaluated-measures","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://cancercontrol.cancer.gov/brp/research/group-evaluated-measures","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"https://cancercontrol.cancer.gov/brp/research/group-evaluated-measures-database","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://gem-measures.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"gem-beta.org","name":"Department of Health and Human Services","redirect":"http://www.gem-beta.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1740965653672,"status":500,"redirect":"http://www.gem-beta.org/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://gem-beta.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"fatherhoodgov.info","name":"Department of Health and Human Services","redirect":"https://fatherhoodgov.info/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://fatherhoodgov.info/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.fatherhoodgov.info","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"everytrycounts.org","name":"Department of Health and Human Services","redirect":"http://www.everytrycounts.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"everytry.org","name":"Department of Health and Human Services","redirect":"http://www.everytry.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"every-try-counts.org","name":"Department of Health and Human Services","redirect":"http://www.every-try-counts.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"docline.org","name":"Department of Health and Human Services","redirect":"http://www.docline.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"directihs.net","name":"Department of Health and Human Services","redirect":"http://www.directihs.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"detroitself.org","name":"Department of Health and Human Services","redirect":"https://detroitself.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://detroitself.org/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://detroitself.org/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"ctsu.org","name":"Department of Health and Human Services","redirect":"https://www.ctsu.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.ctsu.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"csscoperations.com","name":"Department of Health and Human Services","redirect":"https://csscoperations.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://csscoperations.com/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"coronaviruspreventionnetwork.org","name":"Department of Health and Human Services","redirect":"https://www.coronaviruspreventionnetwork.org/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"http://coronaviruspreventionnetwork.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.coronaviruspreventionnetwork.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.coronaviruspreventionnetwork.org/sitemap.xml","source":"robots.txt"},{"time":1730174316792,"status":301,"redirect":"https://coronaviruspreventionnetwork.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1743826667324,"status":200,"redirect":"https://lecampeur.mobi/","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":403,"url":"citdbase.org","name":"Department of Health and Human Services","redirect":"http://www.citdbase.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.citdbase.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"charmmtutorial.org","name":"Department of Health and Human Services","redirect":"http://www.charmmtutorial.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://charmmtutorial.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":500,"redirect":"https://www.charmm.org/wiki/index.php","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://charmmtutorial.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1737430190371,"status":404,"redirect":"https://charmmtutorial.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":404,"redirect":"https://idwlimited.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"charmm.org","name":"Department of Health and Human Services","redirect":"https://academiccharmm.org/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://academiccharmm.org/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://academiccharmm.org/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"http://www.charmm.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":403,"url":"chainonline.org","name":"Department of Health and Human Services","redirect":"http://www.chainonline.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1729037740707,"status":403,"redirect":"https://racebiomed.org/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.chainonline.org/","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":403,"redirect":"https://racebiomed.org/","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"http://www.chainonline.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":500,"redirect":"http://www.chainonline.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"cfsrportal.org","name":"Department of Health and Human Services","redirect":"https://www.cfsrportal.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cdcpartners.org","name":"Department of Health and Human Services","redirect":"http://www.cdcpartners.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"ccwdata.org","name":"Department of Health and Human Services","redirect":"https://www2.ccwdata.org/my.logout.php3?errorcode=19","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www2.ccwdata.org/my.logout.php3?errorcode=19","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://www2.ccwdata.org/my.logout.php3?errorcode=19","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":200,"redirect":"https://ccwdata.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":403,"url":"cahpsusernetwork.net","name":"Department of Health and Human Services","redirect":"https://www.cahpsusernetwork.net/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://cahpsusernetwork.net/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"brainattackcoalition.org","name":"Department of Health and Human Services","redirect":"http://www.brainattackcoalition.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ninds.nih.gov/health-information/patient-caregiver-education/brain-attack-coalition","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"ahrqstg.org","name":"Department of Health and Human Services","redirect":"http://www.ahrqstg.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"ahrqdev.org","name":"Department of Health and Human Services","redirect":"https://ahrqdev.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1743826667324,"status":500,"redirect":"http://www.ahrqdev.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"ahrqadmin.org","name":"Department of Health and Human Services","redirect":"http://www.ahrqadmin.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1730174316792,"status":500,"redirect":"https://www.ahrqadmin.org/","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://ahrqadmin.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"actagainstaids.org","name":"Department of Health and Human Services","redirect":"https://www.actagainstaids.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"americanmadechallenges.org","name":"Department of Energy","redirect":"https://americanmadechallenges.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://americanmadechallenges.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"cultureready.org","name":"Department of Defense","redirect":"http://www.cultureready.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":426,"redirect":"https://cultureready.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"dcsafesurrender.org","name":"Court Services and Offender Supervision Agency","redirect":"https://www.dcsafesurrender.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"http://dcsafesurrender.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"dcsafesurrender.net","name":"Court Services and Offender Supervision Agency","redirect":"https://www.dcsafesurrender.net/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"http://dcsafesurrender.net/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"dcsafesurrender.com","name":"Court Services and Offender Supervision Agency","redirect":"https://www.dcsafesurrender.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"http://dcsafesurrender.com/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"csosadev.net","name":"Court Services and Offender Supervision Agency","redirect":"http://www.csosadev.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"csosa.org","name":"Court Services and Offender Supervision Agency","redirect":"https://www.csosa.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"http://csosa.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"csosa.net","name":"Court Services and Offender Supervision Agency","redirect":"https://www.csosa.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"csosa.mobi","name":"Court Services and Offender Supervision Agency","redirect":"http://www.csosa.mobi/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"http://csosa.mobi/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"csosa.me","name":"Court Services and Offender Supervision Agency","redirect":"http://www.csosa.me/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"http://csosa.me/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"csosa.info","name":"Court Services and Offender Supervision Agency","redirect":"http://www.csosa.info/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"http://csosa.info/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"csosa.com","name":"Court Services and Offender Supervision Agency","redirect":"https://www.csosa.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"csosa.co","name":"Court Services and Offender Supervision Agency","redirect":"http://www.csosa.co/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"http://csosa.co/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"csosa.biz","name":"Court Services and Offender Supervision Agency","redirect":"https://www.csosa.biz/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"http://csosa.biz/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"abmceducation.org","name":"American Battle Monuments Commission","redirect":"http://www.abmceducation.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":200,"redirect":"https://www.marinacoveresort.com/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1742146243896,"status":500,"redirect":"http://www.abmceducation.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"yesprograms.org","name":"Department of State","redirect":"https://yesprograms.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://yesprograms.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"yes-abroad.org","name":"Department of State","redirect":"https://yes-abroad.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://yes-abroad.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ycq2.org","name":"Department of Defense","redirect":"https://ycq2.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://ycq2.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://ycq2.org/wp-sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://ycq2.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://ycq2.org/wp-sitemap.xml","source":"robots.txt"}]},{"status":404,"url":"wibeaches.us","name":"Department of the Interior","redirect":"https://wibeaches.us/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://wibeaches.us/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"westpoint.edu","name":"Department of the Army","redirect":"https://www.westpoint.edu/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"westernmarylandcfc.org","name":"Combined Federal Campaign","redirect":"http://www.westernmarylandcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"westernmacfc.org","name":"Combined Federal Campaign","redirect":"http://www.westernmacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"waterqualitydata.us","name":"Department of Interior","redirect":"https://www.waterqualitydata.us/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.waterqualitydata.us/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"war-on-pineapple.com","name":"Department of Homeland Security","redirect":"http://www.war-on-pineapple.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"voatour.com","name":"U.S. Agency for Global Media","redirect":"https://www.insidevoa.com/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.insidevoa.com/p/8574.html","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.insidevoa.com/p/8574.html","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.voatour.com/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":200,"redirect":"https://www.insidevoa.com/p/8574.html","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"voanews.com","name":"U.S. Agency for Global Media","redirect":"https://www.voanews.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.voanews.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.voanews.com/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.voanews.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.voanews.com/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"vetmedinfo.org","name":"Health and Human Services","redirect":"http://www.vetmedinfo.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"veteranshealthlibrary.org","name":"","redirect":"http://www.veteranshealthlibrary.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"veteranscrisisline.net","name":"Department of Veterans Affairs","redirect":"https://www.veteranscrisisline.net/robots.txt","sitemap":"https://www.veteranscrisisline.net/xmlsitemap","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.veteranscrisisline.net/robots.txt","valid":true,"allowed":true,"sitemap":"https://veteranscrisisline.net/xmlsitemap"},{"time":1740965653672,"status":200,"redirect":"https://www.veteranscrisisline.net/robots.txt","valid":true,"allowed":true,"sitemap":"https://veteranscrisisline.net/xmlsitemap","source":"robots.txt"}]},{"status":404,"url":"vasc.org/erc","name":"National Aeronautics and Space Administration","redirect":"http://vasc.org/erc","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"usuhs.edu","name":"Uniformed Services University of the Health Services","redirect":"https://www.usuhs.edu/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usuhs.edu/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1729037740707,"status":200,"redirect":"https://www.usuhs.edu/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1730174316792,"status":502,"redirect":"https://www.usuhs.edu/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":403,"url":"usu.edu/beetle","name":"U.S. Forest Service","redirect":"https://www.usu.edu/beetle/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usu.edu/beetle/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"ustraveldocs.com","name":"Department of State","redirect":"https://ustraveldocs.com/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://ustraveldocs.com/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"usps.com","name":"U.S. Postal Service","redirect":"https://www.usps.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usps.com/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"usna.edu","name":"United States Naval Academy","redirect":"http://www.usna.edu/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.usna.edu","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://usna.edu/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"usmma.edu","name":"United States Merchant Marine Academy","redirect":"https://www.usmma.edu/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"usmcu.edu","name":"Marine Corps University","redirect":"https://www.usmcu.edu/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usmcu.edu/robots.txt","valid":true,"allowed":true,"sitemap":"https://usmcu.edu/DesktopModules/SiteData/SiteMap.ashx","source":"robots.txt"}]},{"status":404,"url":"usmc-mccs.org","name":"U.S. Marine Corps","redirect":"https://www.usmc-mccs.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usmc-mccs.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"usma.edu","name":"Department of the Army","redirect":"https://www.usma.edu/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"usembassy-china.org.cn","name":"Department of State","redirect":"http://www.usembassy-china.org.cn/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"usdirectexpress.com","name":"Bureau of Fiscal Service","redirect":"https://www.usdirectexpress.com/error-404.html","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usdirectexpress.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"usda.mannlib.cornell.edu","name":"Department of Agriculture","redirect":"https://usda.library.cornell.edu/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://usda.library.cornell.edu/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"uscgnews.com","name":"U.S. Coast Guard","redirect":"https://www.uscgnews.com/robots.txt","sitemap":"https://www.uscgnews.com/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.uscgnews.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.uscgnews.com/sitemap_index.xml"}]},{"status":404,"url":"uscgboating.org","name":"U.S. Coast Guard","redirect":"https://uscgboating.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://uscgboating.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"uscgaux.info","name":"U.S. Coast Guard Auxiliary","redirect":"https://uscgaux.info/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"uscgalaska.com","name":"U.S. Coast Guard","redirect":"http://uscgalaska.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"http://uscgalaska.com/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1742146243896,"status":200,"redirect":"http://uscgalaska.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":500,"redirect":"http://www.uscgalaska.com/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"uscga.edu","name":"U.S. Coast Guard","redirect":"https://uscga.edu/robots.txt","sitemap":"https://uscga.edu/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://uscga.edu/robots.txt","valid":true,"allowed":true,"sitemap":"https://uscga.edu/sitemap_index.xml"}]},{"status":404,"url":"usarmymars.org","name":"US Army Network Enterprise Technology Command","redirect":"https://usarmymars.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://usarmymars.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"http://usarmymars.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":404,"redirect":"https://usarmymars.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":false}]},{"status":500,"url":"usarmyjrotc.com","name":"Department of the Army","redirect":"https://www.usarmyjrotc.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"usafa.edu","name":"Department of the Air Force","redirect":"https://www.usafa.edu/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"us-mission.ch","name":"Department of State","redirect":"https://geneva.usmission.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://geneva.usmission.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"tvars.com","name":"Tennessee Valley Authority","redirect":"https://tvars.com/robots.txt","sitemap":"https://tvars.com/wp-sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://tvars.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://tvars.com/wp-sitemap.xml"}]},{"status":403,"url":"tvaed.com","name":"Tennessee Valley Authority","redirect":"https://www.tva.com/Economic-Development","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":403,"redirect":"https://www.tva.com/Economic-Development","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":302,"redirect":"https://www.tvaed.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"tva.com","name":"Tennessee Valley Authority","redirect":"https://www.tva.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"trumanlibrary.org","name":"National Archives and Records Administration","redirect":"https://trumanlibrary.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.trumanlibrary.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://www.trumanlibrary.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1737430190371,"status":429,"redirect":"https://trumanlibrary.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"http://www.trumanlibrary.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":429,"redirect":"https://trumanlibrary.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"trainex.org","name":"Environmental Protection Agency","redirect":"https://trainex.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.trainex.org","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"todaysmilitary.com","name":"Department of Defense","redirect":"https://www.todaysmilitary.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.todaysmilitary.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.todaysmilitary.com/sitemap.xml"}]},{"status":500,"url":"todaysdocument.tumblr.com","name":"National Archives and Records Administration","redirect":"https://www.todaysdocument.tumblr.com/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://todaysdocument.tumblr.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://todaysdocument.tumblr.com/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://todaysdocument.tumblr.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://todaysdocument.tumblr.com/sitemap.xml","source":"robots.txt"},{"time":1730174316792,"status":200,"redirect":"https://todaysdocument.tumblr.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"tnregionalcfc.org","name":"Combined Federal Campaign","redirect":"http://www.tnregionalcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"theshoalscfc.org","name":"Combined Federal Campaign","redirect":"http://www.theshoalscfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"thecommunityguide.org","name":"Department of Health and Human Services","redirect":"https://stacks.cdc.gov/?cdc_aa_refval=https%3a%2f%2fwww.thecommunityguide.org%2f404.html","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://thecommunityguide.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://thecommunityguide.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"t2health.org","name":"Department of Defense","redirect":"http://www.t2health.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"suncoastcfc.org","name":"Combined Federal Campaign","redirect":"http://www.suncoastcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"suicidepreventionlifeline.org","name":"Substance Abuse and Mental Health Services Administration","redirect":"https://suicidepreventionlifeline.org/robots.txt","sitemap":"https://988lifeline.org/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://988lifeline.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"stripes.com","name":"Department of Defense","redirect":"https://www.stripes.com/robots.txt","sitemap":"https://www.stripes.com/sitemap/sitemap-index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.stripes.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.stripes.com/sitemap/sitemap-index.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.stripes.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.stripes.com/sitemap/sitemap-index.xml","source":"robots.txt"},{"time":1740965653672,"status":200,"redirect":"https://www.stripes.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"spaceappschallenge.org","name":"National Aeronautics and Space Administration","redirect":"https://www.spaceappschallenge.org/robots.txt","sitemap":"https://spaceappschallenge.org/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.spaceappschallenge.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://spaceappschallenge.org/sitemap.xml"}]},{"status":500,"url":"southwestfloridacfc.org","name":"Combined Federal Campaign","redirect":"http://www.southwestfloridacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"socalcfc.org","name":"Combined Federal Campaign","redirect":"http://www.socalcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"smokymountaincfc.org","name":"Combined Federal Campaign","redirect":"http://www.smokymountaincfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"smokeybear.com","name":"U.S. Forest Service","redirect":"https://smokeybear.com/robots.txt","sitemap":"https://smokeybear.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://smokeybear.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://smokeybear.com/sitemap.xml"}]},{"status":200,"url":"smithsonianstore.com","name":"Smithsonian Institution","redirect":"https://www.smithsonianstore.com/robots.txt","sitemap":"https://www.smithsonianstore.com/xmlsitemap.php","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.smithsonianstore.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.smithsonianstore.com/xmlsitemap.php"}]},{"status":500,"url":"smithsonianscience.org","name":"Smithsonian Institution","redirect":"http://www.smithsonianscience.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"smithsonianmag.com","name":"Smithsonian Institution","redirect":"https://www.smithsonianmag.com/robots.txt","sitemap":"https://www.smithsonianmag.com/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.smithsonianmag.com/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.smithsonianmag.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.smithsonianmag.com/sitemap.xml","source":"robots.txt"},{"time":1743826667324,"status":403,"redirect":"https://www.smithsonianmag.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"smithsonianjourneys.org","name":"Smithsonian Institution","redirect":"https://www.smithsonianjourneys.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.smithsonianjourneys.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"smithsonianeducation.org","name":"Smithsonian Institution","redirect":"https://smithsonianeducation.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://smithsonianeducation.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"smithsonianchannel.com","name":"Smithsonian Institution","redirect":"https://www.smithsonianchannel.com/robots.txt","sitemap":"https://www.smithsonianchannel.com/xmlsitemap/episode","history":[{"time":1726617445612,"status":200,"redirect":"https://www.smithsonianchannel.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.smithsonianchannel.com/xmlsitemap/episode"}]},{"status":200,"url":"smithsonian.org","name":"Smithsonian Institution","redirect":"https://www.si.edu/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.si.edu/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.si.edu/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.smithsonian.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"smithsonian.com","name":"Smithsonian Institution","redirect":"https://smithsonian.com/robots.txt","sitemap":"https://smithsonian.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://smithsonian.com/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"si.edu","name":"Smithsonian Institution","redirect":"https://www.si.edu/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.si.edu/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":403,"url":"sftool.ecomedes.com","name":"General Services Administration","redirect":"http://www.sftool.ecomedes.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"sepa-lehighcfc.org","name":"Combined Federal Campaign","redirect":"http://www.sepa-lehighcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"senccfc.org","name":"Combined Federal Campaign","redirect":"http://www.senccfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"scportal.us","name":"Department of Defense","redirect":"http://scportal.us/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"sanangelocfc.org","name":"Combined Federal Campaign","redirect":"http://www.sanangelocfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"rockymountaincfc.org","name":"Combined Federal Campaign","redirect":"http://www.rockymountaincfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"richmondfed.org","name":"Federal Reserve Board","redirect":"https://www.richmondfed.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.richmondfed.org/robots.txt","valid":true,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.richmondfed.org/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"riandsemacfc.org","name":"Combined Federal Campaign","redirect":"http://www.riandsemacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"rferl.org","name":"U.S. Agency for Global Media","redirect":"https://www.rferl.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.rferl.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.rferl.org/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.rferl.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.rferl.org/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"rfa.org","name":"U.S. Agency for Global Media","redirect":"https://www.rfa.org/arc/outboundfeeds/english/robots.txt","sitemap":"https://rfa.org/arc/outboundfeeds/english/sitemap-news/","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.rfa.org/english/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.rfa.org/sitemap.xml.gz"},{"time":1728949876846,"status":200,"redirect":"https://www.rfa.org/english/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.rfa.org/sitemap.xml.gz","source":"robots.txt"},{"time":1730174316792,"status":200,"redirect":"https://www.rfa.org/english/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"rewardsforjustice.net","name":"Department of State","redirect":"https://rewardsforjustice.net/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://rewardsforjustice.net/robots.txt","valid":true,"allowed":false,"sitemap":null}]},{"status":404,"url":"reirs.com","name":"Nuclear Regulatory Commission","redirect":"https://reirs.com/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://reirs.com/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":404,"redirect":"https://reirs.com/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":500,"redirect":"http://www.reirs.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"realwarriors.net","name":"Department of Defense","redirect":"https://www.health.mil/realwarriors","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://realwarriors.net/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"http://www.realwarriors.net/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":404,"redirect":"https://www.realwarriors.net/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":false}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"radiosawa.com","name":"U.S. Agency for Global Media","redirect":"https://www.alhurra.com/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.radiosawa.com/robots.txt","valid":true,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://www.radiosawa.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"r3coastal.com","name":"Federal Emergency Management Agency","redirect":"http://www.r3coastal.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"psoppc.org","name":"Department of Health and Human Services","redirect":"https://psoppc.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://psoppc.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"psoppc.net","name":"Department of Health and Human Services","redirect":"http://www.psoppc.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"psoppc.com","name":"Department of Health and Human Services","redirect":"http://www.psoppc.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"psctool.us","name":"Department of Defense","redirect":"https://psctool.us/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.psctool.us","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"project-open-data.github.io","name":"General Services Administration","redirect":"https://resources.data.gov/","sitemap":null,"valid":true,"allowed":true,"source":"x-robots-tag","history":[{"time":1726617445612,"status":200,"redirect":"https://project-open-data.cio.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://resources.data.gov/","valid":true,"allowed":true,"sitemap":null,"source":"x-robots-tag"},{"time":1730174316792,"status":301,"redirect":"https://project-open-data.cio.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"x-robots-tag"}]},{"status":500,"url":"potomaccfc.org","name":"Combined Federal Campaign","redirect":"http://www.potomaccfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"phpartners.org","name":"Employment and Training Administration","redirect":"http://www.phpartners.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"philadelphiafed.org","name":"Federal Reserve Board","redirect":"https://www.philadelphiafed.org/robots.txt","sitemap":"https://www.philadelphiafed.org/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.philadelphiafed.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.philadelphiafed.org/sitemap.xml"},{"time":1728949876846,"status":404,"redirect":"https://www.philadelphiafed.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":200,"redirect":"https://www.philadelphiafed.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.philadelphiafed.org/sitemap.xml","source":"robots.txt"},{"time":1739509031548,"status":200,"redirect":"https://www.philadelphiafed.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"pepfar.net","name":"Department of State","redirect":"http://www.pepfar.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"peakcfc.com","name":"Combined Federal Campaign","redirect":"http://www.peakcfc.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"pbgc.org","name":"Pension Benefit Guarantee Corporation","redirect":"https://www.pbgc.gov/robots.txt","sitemap":"https://www.pbgc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pbgc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.pbgc.gov/sitemap.xml"}]},{"status":200,"url":"pantex.com","name":"Department of Energy","redirect":"https://pantex.energy.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://pantex.energy.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"pancanal.com","name":"Autoridad del Canal de Panama/Panama Canal Authority","redirect":"https://pancanal.com/robots.txt","sitemap":"https://pancanal.com/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"http://pancanal.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://pancanal.com/sitemap_index.xml"}]},{"status":500,"url":"pacificnwcfc.org","name":"Combined Federal Campaign","redirect":"http://www.pacificnwcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ownyourlimits.org","name":"Department of Defense","redirect":"https://ownyourlimits.org/robots.txt","sitemap":"https://ownyourlimits.org/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://ownyourlimits.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://ownyourlimits.org/sitemap_index.xml"}]},{"status":200,"url":"oup.org","name":"Department of Housing and Urban Development","redirect":"https://oup.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://oup.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"orvcfc.org","name":"Combined Federal Campaign","redirect":"http://www.orvcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"opengovplatform.org","name":"General Services Administration","redirect":"https://www.opengovplatform.org/robots.txt","sitemap":"https://www.opengovplatform.org/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"http://www.opengovplatform.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.opengovplatform.org/sitemap_index.xml"}]},{"status":500,"url":"onslowcountycfc.org","name":"Combined Federal Campaign","redirect":"http://www.onslowcountycfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"onetonline.org","name":"Employment and Training Administration","redirect":"https://www.onetonline.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.onetonline.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"onecpd.info","name":"Department of Housing and Urban Development","redirect":"https://www.hudexchange.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"okaloosa-waltoncfc.org","name":"Combined Federal Campaign","redirect":"http://www.okaloosa-waltoncfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nyccfc.org","name":"Combined Federal Campaign","redirect":"http://www.nyccfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nutmegcfc.org","name":"Combined Federal Campaign","redirect":"http://www.nutmegcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"nsopw.org","name":"Department of Justice","redirect":"https://www.nsopw.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nsopw.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.nsopw.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"http://nsopw.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":200,"redirect":"https://www.nsopw.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://nsopw.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"nps.edu","name":"Naval Postgraduate School","redirect":"https://nps.edu/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://nps.edu/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"northwestfloridacfc.org","name":"Combined Federal Campaign","redirect":"http://www.northwestfloridacfc.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"http://northwestfloridacfc.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"northernnewenglandcfc.org","name":"Combined Federal Campaign","redirect":"http://www.northernnewenglandcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"northernlightscfc.org","name":"Combined Federal Campaign","redirect":"https://www.northernlightscfc.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.northernlightscfc.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"northcoastcfc.org","name":"Combined Federal Campaign","redirect":"http://www.northcoastcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"norcalcfc.org","name":"Combined Federal Campaign","redirect":"http://www.norcalcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"noradsanta.org","name":"North American Aerospace Defense Command","redirect":"https://www.noradsanta.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.noradsanta.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"https://www.noradsanta.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1729037740707,"status":500,"redirect":"https://www.noradsanta.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"newyorkfed.org","name":"Federal Reserve Board","redirect":"https://www.newyorkfed.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.newyorkfed.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"newmexicocfc.org","name":"Combined Federal Campaign","redirect":"http://www.newmexicocfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"newhire-reporting.com","name":"Administration for Children and Families","redirect":"https://newhire-reporting.com/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://newhire-reporting.com/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"nevadacfc.org","name":"Combined Federal Campaign","redirect":"http://www.nevadacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nepacfc.org","name":"Combined Federal Campaign","redirect":"http://www.nepacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"nefl-sega-cfc.org","name":"Combined Federal Campaign","redirect":"https://nefl-sega-cfc.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":301,"redirect":"https://www.nefl-sega-cfc.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":404,"redirect":"https://www.nefl-sega-cfc.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":500,"redirect":"https://pafikotasingaraja.info/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"necfuture.com","name":"Federal Railroad Administration","redirect":"https://necfuture.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"http://necfuture.com/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"neabigread.org","name":"National Endowment for the Arts","redirect":"https://neabigread.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://neabigread.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":404,"redirect":"https://neabigread.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ndu.edu","name":"National Defense University","redirect":"https://www.ndu.edu/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.ndu.edu/robots.txt","valid":true,"allowed":true,"sitemap":"https://ndu.edu/DesktopModules/SiteData/SiteMap.ashx","source":"robots.txt"}]},{"status":500,"url":"ncwvcfc.org","name":"Combined Federal Campaign","redirect":"http://www.ncwvcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"navyreserve.com","name":"Department of the Navy","redirect":"https://www.navy.com/joining/ways-to-join/reserve","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.navy.com/joining/ways-to-join/reserve","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.navy.com/joining/ways-to-join/reserve","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.navyreserve.com/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":200,"redirect":"https://www.navy.com/joining/ways-to-join/reserve","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":403,"redirect":"https://www.navy.com/joining/ways-to-join/reserve","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"navy.com","name":"Department of the Navy","redirect":"https://www.navy.com/robots.txt","sitemap":"https://www.navy.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.navy.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.navy.com/sitemap.xml"},{"time":1734148194754,"status":200,"redirect":"https://www.navy.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.navy.com/sitemap.xml","source":"robots.txt"},{"time":1737430190371,"status":403,"redirect":"https://www.navy.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"nationalguard.com","name":"U.S. National Guard","redirect":"https://nationalguard.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://nationalguard.com/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"nationalatlas.org","name":"Department of Interior","redirect":"http://www.nationalatlas.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nationalatlas.com","name":"Department of Interior","redirect":"http://www.nationalatlas.com/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":301,"redirect":"https://nationalmap.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nasa-usa.de","name":"National Aeronautical and Space Administration","redirect":"https://www.nasa.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nasa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"nagb.org","name":"National Assessment Governing Board","redirect":"https://www.nagb.gov/robots.txt","sitemap":"https://www.nagb.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nagb.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.nagb.gov/sitemap.xml"}]},{"status":200,"url":"myskillsmyfuture.org","name":"Employment and Training Administration","redirect":"https://www.myskillsmyfuture.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.myskillsmyfuture.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"mymcx.com","name":"U.S. Marine Corps","redirect":"https://www.mymcx.com/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.mymcx.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"myfuture.com","name":"Department of Defense","redirect":"https://myfuture.com/robots.txt","sitemap":"https://myfuture.com/sitemap-index.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://myfuture.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://myfuture.com/sitemap.xml"},{"time":1740965653672,"status":200,"redirect":"https://myfuture.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://myfuture.com/sitemap.xml","source":"robots.txt"}]},{"status":429,"url":"myedaccount.com","name":"Department of Education","redirect":"http://myedaccount.com/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":429,"redirect":"http://myedaccount.com/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"http://myedaccount.com/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"},{"time":1740965653672,"status":200,"redirect":"https://myedaccount.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":429,"url":"moon-cal.org","name":"U.S. Geological Survey","redirect":"http://www.moon-cal.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":429,"redirect":"http://moon-cal.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":500,"redirect":"http://www.moon-cal.org/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":429,"redirect":"https://moon-cal.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":429,"redirect":"https://moon-cal.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"moneyfactory.com","name":"Bureau of Engraving and Printing","redirect":"https://www.moneyfactory.com/robots.txt","sitemap":"https://www.moneyfactory.com/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.moneyfactory.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.moneyfactory.com/sitemap_index.xml"}]},{"status":200,"url":"minneapolisfed.org","name":"Federal Reserve Board","redirect":"https://www.minneapolisfed.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.minneapolisfed.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"midlandsareacfc.org","name":"Combined Federal Campaign","redirect":"http://www.midlandsareacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"michcfc.org","name":"Combined Federal Campaign","redirect":"http://www.michcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"micanaldepanama.com","name":"Autoridad del Canal de Panama/Panama Canal Authority","redirect":"https://pancanal.com/robots.txt","sitemap":"https://pancanal.com/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://pancanal.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://pancanal.com/sitemap_index.xml"},{"time":1728949876846,"status":500,"redirect":"https://calculadora.pancanal.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"miamivalleycfc.org","name":"Combined Federal Campaign","redirect":"http://www.miamivalleycfc.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"http://www.miamivalleycfc.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"metrodenvercfc.org","name":"Combined Federal Campaign","redirect":"http://www.metrodenvercfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"mdacareers.com","name":"Missile Defense Agency","redirect":"http://www.mdacareers.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":301,"url":"mcmregistration.com","name":"U.S. Marine Corps","redirect":"https://mcmregistration.com/404.html","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"martinoticias.com","name":"U.S. Agency for Global Media","redirect":"https://www.martinoticias.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.martinoticias.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.martinoticias.com/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.martinoticias.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.martinoticias.com/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"marshallcenter.org","name":"George C. Marshall European Center for Security Studies","redirect":"http://www.marshallcenter.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"marines.com","name":"U.S. Marine Corps","redirect":"https://www.marines.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.marines.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.marines.com/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.marines.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.marines.com/sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"marinemarathon.com","name":"U.S. Marine Corps","redirect":"https://www.marinemarathon.com/robots.txt","sitemap":"https://www.marinemarathon.com/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.marinemarathon.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.marinemarathon.com/sitemap_index.xml"}]},{"status":200,"url":"marineforlife.org","name":"U.S. Marine Corps","redirect":"https://www.usmc-mccs.org/marine-family-support/transition-readiness/marine-for-life-network","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usmc-mccs.org/marine-family-support/transition-readiness/marine-for-life-network","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usmc-mccs.org/marine-family-support/transition-readiness/marine-for-life-network","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://www.marineforlife.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"maricopacfc.org","name":"Combined Federal Campaign","redirect":"https://www.maricopacfc.org/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://maricopacfc.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.maricopacfc.org/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.maricopacfc.org/robots.txt/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"manufacturingusa.com","name":"National Institute of Standards and Technology","redirect":"https://www.manufacturingusa.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.manufacturingusa.com/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"makersinspaces.org","name":"Department of State","redirect":"http://www.makersinspaces.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"livingmemorialsproject.net","name":"U.S. Forest Service","redirect":"https://www.livingmemorialsproject.net/robots.txt","sitemap":"https://www.livingmemorialsproject.net/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://livingmemorialsproject.net/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.livingmemorialsproject.net/sitemap.xml"}]},{"status":404,"url":"literacydirectory.org","name":"Department of Education","redirect":"https://literacydirectory.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://literacydirectory.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"http://literacydirectory.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":404,"redirect":"https://literacydirectory.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":false}]},{"status":500,"url":"lifeasamarine.com","name":"U.S. Marine Corps","redirect":"http://www.lifeasamarine.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"leopold.wilderness.net","name":"U.S. Forest Service","redirect":"https://leopold.wilderness.net/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://leopold.wilderness.net/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"lbjlibrary.org","name":"National Archives and Records Administration","redirect":"https://lbjlibrary.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://lbjlibrary.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"ktcfc.org","name":"Combined Federal Campaign","redirect":"http://www.ktcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"knowyouroptions.com","name":"Federal National Mortgage Association","redirect":"https://www.knowyouroptions.com/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.knowyouroptions.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://singlefamily.fanniemae.com/sitemap.xml"},{"time":1740965653672,"status":200,"redirect":"https://www.knowyouroptions.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://singlefamily.fanniemae.com/sitemap.xml","source":"robots.txt"}]},{"status":404,"url":"kcp.com","name":"Department of Energy","redirect":"https://kcnsc.doe.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://kcnsc.doe.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"kansascfc.org","name":"Combined Federal Campaign","redirect":"http://www.kansascfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"johnsoncity-mountainhomecfc.org","name":"Combined Federal Campaign","redirect":"http://www.johnsoncity-mountainhomecfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"jlab.org","name":"Department of Energy","redirect":"https://www.jlab.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.jlab.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":403,"url":"jfklibrary.org","name":"National Archives and Records Administration","redirect":"https://www.jfklibrary.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.jfklibrary.org/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://www.jfklibrary.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"jamrs.org","name":"Department of Defense","redirect":"https://jamrs.defense.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://jamrs.defense.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://jamrs.org/DesktopModules/SiteData/SiteMap.ashx"},{"time":1728949876846,"status":200,"redirect":"https://jamrs.defense.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://jamrs.org/DesktopModules/SiteData/SiteMap.ashx","source":"robots.txt"}]},{"status":500,"url":"inwcfc.org","name":"Combined Federal Campaign","redirect":"http://www.inwcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"intermountaincfc.org","name":"Combined Federal Campaign","redirect":"https://intermountaincfc.org/robots.txt","sitemap":"http://intermountaincfc.org/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"http://intermountaincfc.org/robots.txt","valid":true,"allowed":true,"sitemap":"http://intermountaincfc.org/sitemap_index.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"insidevoa.com","name":"U.S. Agency for Global Media","redirect":"https://www.insidevoa.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.insidevoa.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.insidevoa.com/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.insidevoa.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.insidevoa.com/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"informationrx.org","name":"Health and Human Services","redirect":"http://www.informationrx.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"illowacfc.org","name":"Combined Federal Campaign","redirect":"http://www.illowacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"iipdigital.ait.org.tw","name":"Department of State","redirect":"http://www.iipdigital.ait.org.tw/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"igs.org","name":"National Aeronautics and Space Administration","redirect":"https://igs.org/robots.txt","sitemap":"https://igs.org/wp-sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://igs.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://igs.org/wp-sitemap.xml"}]},{"status":500,"url":"huduser.org","name":"Department of Housing and Urban Development","redirect":"https://www.huduser.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.huduser.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://www.huduser.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"hudnsphelp.info","name":"Department of Housing and Urban Development","redirect":"https://hudnsphelp.info/robots.txt","sitemap":"https://hudnsphelp.info/wp-sitemap.xml","history":[{"time":1726617445612,"status":404,"redirect":"http://hudnsphelp.info/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://hudnsphelp.info/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"hudhre.info","name":"Department of Housing and Urban Development","redirect":"https://hudhre.info/robots.txt","sitemap":"https://hudhre.info/wp-sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://hudhre.info/robots.txt","valid":true,"allowed":true,"sitemap":"https://hudhre.info/wp-sitemap.xml"},{"time":1730174316792,"status":403,"redirect":"https://hookerdunham.org/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"hudhomestore.com","name":"Department of Housing and Urban Development","redirect":"https://www.hudhomestore.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":"https://www.hudhomestore.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":500,"redirect":"https://www.hudhomestore.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":404,"url":"hudexchange.info","name":"Department of Housing and Urban Development","redirect":"https://www.hudexchange.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"hubblesite.org","name":"National Aeronautics and Space Administration","redirect":"https://www.hubblesite.org/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":403,"redirect":"https://hubblesite.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"hsdl.org","name":"Department of Homeland Security","redirect":"https://www.hsdl.org/c/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://hsdl.org/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.hsdl.org","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":200,"redirect":"https://www.hsdl.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"homelandsecurity.conferenceon.tv","name":"Department of Homeland Security","redirect":"http://www.homelandsecurity.conferenceon.tv/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"heartofthemidlandscfc.org","name":"Combined Federal Campaign","redirect":"http://www.heartofthemidlandscfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"heartofalabamacfc.org","name":"Combined Federal Campaign","redirect":"http://www.heartofalabamacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"heartlandcfc.org","name":"Combined Federal Campaign","redirect":"https://heartlandcfc.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"greencountrycfc.org","name":"Combined Federal Campaign","redirect":"http://www.greencountrycfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"greaterwicfc.org","name":"Combined Federal Campaign","redirect":"http://www.greaterwicfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"greatermscfc.org","name":"Combined Federal Campaign","redirect":"http://www.greatermscfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"greaterhartfordcfc.org","name":"Combined Federal Campaign","redirect":"http://www.greaterhartfordcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"gopcfc.org","name":"Combined Federal Campaign","redirect":"http://www.gopcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"godirect.org","name":"Federal Reserve Board","redirect":"https://www.godirect.gov/gpw/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.godirect.gov/maintenance/redirect.html","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"gocoastguard.com","name":"U.S. Coast Guard","redirect":"https://gocoastguard.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"https://www.gocoastguard.com/","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":200,"redirect":"https://gocoastguard.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1730174316792,"status":500,"redirect":"http://www.gocoastguard.com/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"gocivilairpatrol.com","name":"U.S. Air Force Auxiliary","redirect":"https://www.gocivilairpatrol.com/robots.txt","sitemap":"https://www.gocivilairpatrol.com/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.gocivilairpatrol.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.gocivilairpatrol.com/sitemap.xml"}]},{"status":200,"url":"goarmy.com","name":"Department of the Army","redirect":"https://www.goarmy.com/robots.txt","sitemap":"https://www.goarmy.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.goarmy.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.goarmy.com/sitemap.xml"}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"goang.com","name":"Department of the Air Force","redirect":"https://www.airforce.com/goang/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.airforce.com/goang/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"glnpo.net","name":"Environmental Protection Agency","redirect":"http://www.glnpo.net/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"getsmartaboutdrugs.com","name":"Drug Enforcement Agency","redirect":"https://www.getsmartaboutdrugs.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.getsmartaboutdrugs.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"georgewbushlibrary.smu.edu","name":"National Archives and Records Administration","redirect":"https://www.georgewbushlibrary.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.georgewbushlibrary.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"genomereference.org","name":"Department of Health and Human Services","redirect":"https://www.ncbi.nlm.nih.gov/grc","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ncbi.nlm.nih.gov/grc","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"https://www.genomereference.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://genomereference.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"gatewaycfc.org","name":"Combined Federal Campaign","redirect":"https://nexuscommunity.org/","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://gatewaycfc.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://nexuscommunity.org/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://gatewaycfc.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"ftcampbellareacfc.org","name":"Combined Federal Campaign","redirect":"http://www.ftcampbellareacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"fsafeds.com","name":"Office of Personnel Management","redirect":"https://www.fsafeds.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":403,"redirect":"https://www.fsafeds.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.fsafeds.com/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"freefilefillableforms.com","name":"Internal Revenue Service","redirect":"https://freefilefillableforms.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.irs.gov/e-file-providers/free-file-fillable-forms","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.irs.gov/e-file-providers/free-file-fillable-forms","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":200,"redirect":"https://freefilefillableforms.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1734148194754,"status":200,"redirect":"https://www.irs.gov/e-file-providers/free-file-fillable-forms","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"freddiemac.com","name":"Federal Home Loan Mortgage Corporation","redirect":"https://www.freddiemac.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.freddiemac.com/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"frbsf.org","name":"Federal Reserve Board","redirect":"https://www.frbsf.org/robots.txt","sitemap":"https://www.frbsf.org/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.frbsf.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"frbatlanta.org","name":"Federal Reserve Board","redirect":"https://www.atlantafed.org/robots.txt","sitemap":"https://www.atlantafed.org/sitemap","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"https://www.atlantafed.org/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":500,"redirect":"https://www.atlantafed.org/","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":200,"redirect":"https://www.atlantafed.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.atlantafed.org/sitemap","source":"robots.txt"},{"time":1743826667324,"status":500,"redirect":"https://www.atlantafed.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"forestryimages.org","name":"U.S. Forest Service","redirect":"https://www.forestryimages.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.forestryimages.org/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"https://www.forestryimages.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":404,"url":"fmjfee.com","name":"U.S. Citizenship and Immigration Service","redirect":"https://www.fmjfee.com/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://fmjfee.com/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"firelab.org","name":"U.S. Forest Service","redirect":"https://firelab.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://firelab.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"findtbresources.org","name":"Centers for Disease Control and Prevention","redirect":"http://www.findtbresources.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"fhlbtopeka.com","name":"Interagency Collaboration","redirect":"https://www.fhlbtopeka.com/robots.txt","sitemap":"https://www.fhlbtopeka.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fhlbtopeka.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fhlbtopeka.com/sitemap.xml"}]},{"status":200,"url":"fhlbsf.com","name":"Interagency Collaboration","redirect":"https://www.fhlbsf.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://fhlbsf.com/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"fhlbsea.com","name":"Interagency Collaboration","redirect":"https://fhlbsea.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://fhlbsea.com/robots.txt","valid":true,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"http://fhlbsea.com/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"fhlbny.com","name":"Interagency Collaboration","redirect":"https://www.fhlbny.com/robots.txt","sitemap":"https://www.fhlbny.com/sitemap_index.xml","history":[{"time":1726617445612,"status":404,"redirect":"http://fhlbny.com/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fhlbi.com","name":"Interagency Collaboration","redirect":"https://www.fhlbi.com/robots.txt","sitemap":"https://www.fhlbi.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fhlbi.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fhlbi.com/sitemap.xml"}]},{"status":403,"url":"fhlbdm.com","name":"Interagency Collaboration","redirect":"https://www.fhlbdm.com/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://fhlbdm.com/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":200,"redirect":"https://fhlbdm.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"fhlbcin.com","name":"Interagency Collaboration","redirect":"https://www.fhlbcin.com/my.logout.php3?errorcode=19","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.fhlbcin.com/my.logout.php3?errorcode=19","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":302,"redirect":"https://www.fhlbcin.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"fhlbc.com","name":"Interagency Collaboration","redirect":"https://www.fhlbc.com/robots.txt","sitemap":"https://www.fhlbc.com/sitemap-index.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fhlbc.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fhlbc.com/sitemap-index.xml"}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"fhlbboston.com","name":"Interagency Collaboration","redirect":"https://www.fhlbboston.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"valid":false,"source":false,"allowed":false,"status":500,"url":"fhlbatl.com","name":"Interagency Collaboration","redirect":"http://fhlbatl.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":404,"redirect":"http://www.fhlbatl.com/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"fhlbanks.com","name":"Interagency Collaboration","redirect":"https://fhlbanks.com/robots.txt","sitemap":"https://fhlbanks.com/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://fhlbanks.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://fhlbanks.com/sitemap_index.xml"}]},{"status":200,"url":"fhlb.com","name":"Interagency Collaboration","redirect":"https://www.fhlb.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fhlb.com/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"fhlb-pgh.com","name":"Interagency Collaboration","redirect":"https://www.fhlb-pgh.com/robots.txt","sitemap":"https://www.fhlb-pgh.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fhlb-pgh.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fhlb-pgh.com/sitemap.xml"}]},{"status":200,"url":"fhlb-of.com","name":"Interagency Collaboration","redirect":"https://fhlb-of.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://fhlb-of.com/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"federalreserveeducation.org","name":"Federal Reserve Board","redirect":"https://www.federalreserveeducation.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"federalrelay.us","name":"General Services Administration","redirect":"https://www.tmobileaccess.com/services/captions-for-federal-government","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.tmobileaccess.com/services/captions-for-federal-government","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.tmobileaccess.com/services/captions-for-federal-government","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.federalrelay.us/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"federallabs.org","name":"Interagency Collaboration","redirect":"https://federallabs.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://federallabs.org/robots.txt","valid":true,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://federallabs.org/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":200,"redirect":"https://federallabs.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":200,"redirect":"https://federallabs.org/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"fdrlibrary.marist.edu","name":"National Archives and Records Administration","redirect":"https://www.fdrlibrary.org/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.fdrlibrary.org/","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":500,"redirect":"https://fdrlibrary.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fd.org","name":"U.S. Courts","redirect":"https://www.fd.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://fd.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"fcc.us","name":"Federal Communications Commission","redirect":"http://www.fcc.us/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"fbiacademy.edu","name":"Federal Bureau of Investigation","redirect":"http://www.fbiacademy.edu/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"farmermac.com","name":"Federal Agricultural Mortgage Corporation","redirect":"https://www.farmermac.com/robots.txt","sitemap":"http://www.farmermac.com/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.farmermac.com/robots.txt","valid":true,"allowed":true,"sitemap":"http://www.farmermac.com/sitemap_index.xml"}]},{"status":200,"url":"farmcreditbank.com","name":"Farm Credit Administration","redirect":"https://www.farmcreditbank.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.farmcreditbank.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fanniemae.com","name":"Federal National Mortgage Association","redirect":"https://www.fanniemae.com/robots.txt","sitemap":"https://singlefamily.fanniemae.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fanniemae.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://singlefamily.fanniemae.com/sitemap.xml"}]},{"status":403,"url":"fairhousingfirst.org","name":"Department of Housing and Urban Development","redirect":"https://www.hud.gov/program_offices/fair_housing_equal_opp/accessibility_first_home","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.hud.gov/program_offices/fair_housing_equal_opp/accessibility_first_homerobots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":404,"redirect":"https://www.hud.gov/program_offices/fair_housing_equal_opp/accessibility_first_homerobots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":302,"redirect":"http://www.fairhousingfirst.org/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":404,"redirect":"https://www.hud.gov/program_offices/fair_housing_equal_opp/accessibility_first_homerobots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":302,"redirect":"http://www.fairhousingfirst.org/","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":500,"redirect":"http://www.fairhousingfirst.org/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://www.hud.gov/program_offices/fair_housing_equal_opp/accessibility_first_homerobots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"escarosacfc.org","name":"Combined Federal Campaign","redirect":"http://www.escarosacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ertvideo.org","name":"Environmental Protection Agency","redirect":"http://www.ertvideo.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ertsupport.org","name":"Environmental Protection Agency","redirect":"http://www.ertsupport.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"epcfc.org","name":"Combined Federal Campaign","redirect":"http://www.epcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"epaosc.org","name":"Environmental Protection Agency","redirect":"https://response.epa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.epaosc.org","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"epancic.org","name":"Environmental Protection Agency","redirect":"http://www.epancic.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"epa.maps.arcgis.com","name":"Environmental Protection Agency","redirect":"https://epa.maps.arcgis.com/robots.txt","sitemap":"http://static.arcgis.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://epa.maps.arcgis.com/robots.txt","valid":true,"allowed":true,"sitemap":"http://static.arcgis.com/sitemap.xml"}]},{"status":404,"url":"enviroflash.info","name":"Environmental Protection Agency","redirect":"https://www.enviroflash.info/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.enviroflash.info/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"elcivicsonline.org","name":"Department of Education","redirect":"https://elcivicsonline.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://elcivicsonline.org/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1729037740707,"status":200,"redirect":"https://elcivicsonline.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1737430190371,"status":404,"redirect":"https://elcivicsonline.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"efts.scportal.us","name":"Department of Defense","redirect":"http://efts.scportal.us/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"easternmasscfc.org","name":"Combined Federal Campaign","redirect":"http://www.easternmasscfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"earthsystemmodeling.org","name":"National Aeronautics and Space Administration","redirect":"https://earthsystemmodeling.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://earthsystemmodeling.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.earthsystemmodeling.org/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"http://earthsystemmodeling.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.earthsystemmodeling.org/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"earthkam.ucsd.edu","name":"National Aeronautics and Space Administration","redirect":"http://www.earthkam.ucsd.edu/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"e2.gov.cwtsatotravel.com","name":"General Services Administration","redirect":"https://e2.gov.cwtsatotravel.com/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://e2.gov.cwtsatotravel.com/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"dvidshub.net","name":"Department of Defense","redirect":"https://www.dvidshub.net/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.dvidshub.net/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"dvbic.org","name":"Department of Defense","redirect":"https://dvbic.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://dvbic.org/robots.txt","valid":true,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"http://dvbic.org/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"dodstarbase.org","name":"Department of Defense","redirect":"https://dodstarbase.org/robots.txt","sitemap":"https://dodstarbase.org/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://dodstarbase.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://dodstarbase.org/sitemap.xml"}]},{"status":200,"url":"dodsbir.net","name":"Department of Defense","redirect":"http://dodsbir.net/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"dodea.edu","name":"Department of Defense","redirect":"https://www.dodea.edu/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.dodea.edu/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"dodccrp.org","name":"Department of Defense","redirect":"http://www.dodccrp.org/robots.txt","sitemap":"http://www.dodccrp.org/html4/sitemap.html","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://dodccrp.org/robots.txt","valid":true,"allowed":true,"sitemap":"http://www.dodccrp.org/html4/sitemap.html"}]},{"status":403,"url":"docsteach.org","name":"National Archives and Records Administration","redirect":"https://www.docsteach.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"digitalvaults.org","name":"National Archives and Records Administration","redirect":"https://digitalvaults.org/robots.txt","sitemap":"https://digitalvaults.org/wp-sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"http://digitalvaults.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://digitalvaults.org/sitemap.xml"},{"time":1728949876846,"status":500,"redirect":"https://digitalvaults.org/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://digitalvaults.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://digitalvaults.org/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"dhs-summit.us","name":"Department of Homeland Security","redirect":"http://www.dhs-summit.us/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"dfwmetroplexcfc.org","name":"Combined Federal Campaign","redirect":"http://www.dfwmetroplexcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"defensetv.tv","name":"Department of Defense","redirect":"http://www.defensetv.tv/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"deamuseum.org","name":"Drug Enforcement Agency","redirect":"https://museum.dea.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://museum.dea.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.deamuseum.org","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.deamuseum.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"dallasfed.org","name":"Federal Reserve Board","redirect":"https://www.dallasfed.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.dallasfed.org/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.dallasfed.org","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"d13cgaux.com","name":"U.S. Coast Guard Auxiliary","redirect":"http://www.d13cgaux.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"d11nuscgaux.info","name":"U.S. Coast Guard Auxiliary","redirect":"http://d11nuscgaux.info/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"http://d11nuscgaux.info/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":404,"redirect":"http://d11nuscgaux.info/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1734148194754,"status":500,"redirect":"https://wow.uscgaux.info/content.php?unit=113","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"cybercemetery.unt.edu","name":"Quasigovernmental","redirect":"https://cybercemetery.unt.edu/robots.txt/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://cybercemetery.unt.edu/robots.txt/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"cvand29palmscfc.org","name":"Combined Federal Campaign","redirect":"http://www.cvand29palmscfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ctandwmacfc.org","name":"Combined Federal Campaign","redirect":"http://www.ctandwmacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cowvcfc.org","name":"Combined Federal Campaign","redirect":"http://www.cowvcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cooperhewitt.org","name":"Smithsonian Institution","redirect":"https://www.cooperhewitt.org/robots.txt","sitemap":"http://www.cooperhewitt.org/sitemap.xml","history":[{"time":1726617445612,"status":404,"redirect":"http://cooperhewitt.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"cooperativepatentclassification.org","name":"U.S. Patent and Trademark Office","redirect":"https://www.cooperativepatentclassification.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cooperativepatentclassification.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"commissaries.com","name":"Defense Commissary Agency","redirect":"https://www.commissaries.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"columbusbenningcfc.org","name":"Combined Federal Campaign","redirect":"http://www.columbusbenningcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"cobank.com","name":"Farm Credit Administration","redirect":"https://cobank.com/robots.txt","sitemap":"https://cobank.com:443/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://cobank.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://cobank.com:443/sitemap.xml"}]},{"status":500,"url":"coastalcarolinacfc.org","name":"Combined Federal Campaign","redirect":"http://www.coastalcarolinacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"coastalbendareacfc.org","name":"Combined Federal Campaign","redirect":"http://www.coastalbendareacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"clu-in.org","name":"Environmental Protection Agency","redirect":"https://clu-in.org/robots.txt","sitemap":"https://clu-in.org/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.clu-in.org","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"clevelandfed.org","name":"Federal Reserve Board","redirect":"https://www.clevelandfed.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.clevelandfed.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"cldcentral.usalearning.net","name":"Office of Personnel Management","redirect":"https://cldcentral.usalearning.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.cldcentral.usalearning.net","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":500,"url":"cipm.info","name":"National Science Foundation","redirect":"http://cipm.info/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://cipm.ncsu.edu/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"http://www.cipm.info/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"cidi.org","name":"U.S. Agency for International Development","redirect":"https://cidi.usaid.gov/bhacidi","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://cidi.usaid.gov/bhacidi","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://cidi.usaid.gov/bhacidi","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.cidi.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"chicagofed.org","name":"Federal Reserve Board","redirect":"https://www.chicagofed.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.chicagofed.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"chicagocfc.net","name":"Combined Federal Campaign","redirect":"http://www.chicagocfc.net/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":404,"redirect":"https://www.chicagocfc.net/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"chemcatbio.org","name":"Department of Energy","redirect":"https://www.chemcatbio.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.chemcatbio.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"chattanoogacfc.org","name":"Combined Federal Campaign","redirect":"http://www.chattanoogacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"cge.concursolutions.com","name":"General Services Administration","redirect":"https://usg.concursolutions.com/govui","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://usg.concursolutions.com/govui","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://usg.concursolutions.com/govui","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://cge.concursolutions.com/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"cgaux.org","name":"U.S. Coast Guard Auxiliary","redirect":"https://cgaux.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://cgaux.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":409,"url":"cga.edu","name":"U.S. Coast Guard","redirect":"http://www.cga.edu/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfe-dmha.org","name":"Department of Defense","redirect":"http://www.cfe-dmha.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcwnc.com","name":"Combined Federal Campaign","redirect":"http://www.cfcwnc.com/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcwesternmontana.org","name":"Combined Federal Campaign","redirect":"http://www.cfcwesternmontana.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcwesternmichigan.org","name":"Combined Federal Campaign","redirect":"http://www.cfcwesternmichigan.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cfcvp.org","name":"Combined Federal Campaign","redirect":"https://cfcvp.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cfcvp.org/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cfcvp.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1734148194754,"status":200,"redirect":"https://cfcvp.org/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"},{"time":1737430190371,"status":200,"redirect":"https://cfcvp.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":500,"redirect":"http://www.cfcvp.org/","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://www.cfcvp.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1742146243896,"status":200,"redirect":"https://www.luminacity.co.id/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1743826667324,"status":500,"redirect":"http://cfcvp.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"cfctexasgulfcoast.org","name":"Combined Federal Campaign","redirect":"http://www.cfctexasgulfcoast.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcsnj.org","name":"Combined Federal Campaign","redirect":"http://www.cfcsnj.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"cfcshr.org","name":"Combined Federal Campaign","redirect":"https://www.cfcshr.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcsemi.org","name":"Combined Federal Campaign","redirect":"http://www.cfcsemi.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcsanantonio.org","name":"Combined Federal Campaign","redirect":"http://www.cfcsanantonio.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcredrivervalley.org","name":"Combined Federal Campaign","redirect":"http://www.cfcredrivervalley.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcofthecsra.org","name":"Combined Federal Campaign","redirect":"http://www.cfcofthecsra.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcofgreatersocal.org","name":"Combined Federal Campaign","redirect":"http://www.cfcofgreatersocal.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcofcva.org","name":"Combined Federal Campaign","redirect":"http://www.cfcofcva.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcnps.org","name":"Combined Federal Campaign","redirect":"http://www.cfcnps.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcnorthtexas.org","name":"Combined Federal Campaign","redirect":"http://www.cfcnorthtexas.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcnnj.org","name":"Combined Federal Campaign","redirect":"http://www.cfcnnj.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcnexus.org","name":"Combined Federal Campaign","redirect":"http://www.cfcnexus.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"cfcnca.org","name":"Combined Federal Campaign","redirect":"https://givecfc.org/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://givecfc.org/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://givecfc.org/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.cfcnca.org/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":200,"redirect":"https://givecfc.org/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":500,"redirect":"https://givecfc.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"cfcmontereysantacruz.org","name":"Combined Federal Campaign","redirect":"http://www.cfcmontereysantacruz.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcmidsouth.org","name":"Combined Federal Campaign","redirect":"http://www.cfcmidsouth.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"cfcmaine.org","name":"Combined Federal Campaign","redirect":"https://www.cfcmaine.org/robots.txt","sitemap":"https://www.cfcmaine.org/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://cfcmaine.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.cfcmaine.org/sitemap.xml"}]},{"status":500,"url":"cfcindiana.org","name":"Combined Federal Campaign","redirect":"http://www.cfcindiana.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcgreaternc.org","name":"Combined Federal Campaign","redirect":"http://www.cfcgreaternc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcgreaterarkansas.org","name":"Combined Federal Campaign","redirect":"http://www.cfcgreaterarkansas.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcgno.org","name":"Combined Federal Campaign","redirect":"http://www.cfcgno.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"cfcgive.org","name":"Combined Federal Campaign","redirect":"https://cfcgive.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1729037740707,"status":500,"redirect":"http://www.cfcgive.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"cfcforthood.org","name":"Combined Federal Campaign","redirect":"http://www.cfcforthood.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfccoastalga.org","name":"Combined Federal Campaign","redirect":"http://www.cfccoastalga.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcaz.org","name":"Combined Federal Campaign","redirect":"http://www.cfcaz.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfcaaga.org","name":"Combined Federal Campaign","redirect":"http://www.cfcaaga.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cfc-centralok.org","name":"Combined Federal Campaign","redirect":"http://www.cfc-centralok.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cf-cfc.org","name":"Combined Federal Campaign","redirect":"http://www.cf-cfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"centraltexascfc.org","name":"Combined Federal Campaign","redirect":"http://www.centraltexascfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"centralpacfc.org","name":"Combined Federal Campaign","redirect":"http://www.centralpacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"centralcaliforniacfc.org","name":"Combined Federal Campaign","redirect":"http://www.centralcaliforniacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cdse.edu","name":"Department of Defense","redirect":"https://www.cdse.edu/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cdse.edu/robots.txt","valid":true,"allowed":true,"sitemap":"https://cdse.edu/DesktopModules/SiteData/SiteMap.ashx","source":"robots.txt"}]},{"status":404,"url":"cdcnpin.org","name":"Centers for Disease Control and Prevention","redirect":"http://cdcnpin.org/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://cdcnpin.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"cbp-ntep.info","name":"Customs and Border Patrol","redirect":"http://www.cbp-ntep.info/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cbacfc.org","name":"Combined Federal Campaign","redirect":"https://cbacfc.org/robots.txt","sitemap":"https://cbacfc.org/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://cbacfc.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://cbacfc.org/sitemap_index.xml"}]},{"status":202,"url":"careeronestop.org","name":"Employment and Training Administration","redirect":"https://www.careeronestop.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1734148194754,"status":403,"redirect":"https://www.careeronestop.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":202,"url":"careerinfonet.org","name":"Employment and Training Administration","redirect":"https://www.careeronestop.org/Toolkit/ACINet.aspx","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":403,"redirect":"https://www.careeronestop.org/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.careerinfonet.org/","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":403,"redirect":"https://www.careeronestop.org/Toolkit/ACINet.aspx","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":202,"redirect":"https://www.careeronestop.org/Toolkit/ACINet.aspx","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.careerinfonet.org/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"capmembers.com","name":"U.S. Air Force Auxiliary","redirect":"https://www.gocivilairpatrol.com/robots.txt","sitemap":"https://www.gocivilairpatrol.com/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.gocivilairpatrol.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.gocivilairpatrol.com/sitemap.xml"}]},{"status":200,"url":"buyaccessible.net","name":"General Services Administration","redirect":"http://www.buyaccessible.net/robots.txt","sitemap":"http://www.buyaccessible.net/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://www.buyaccessible.net/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.buyaccessible.net","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"bushlibrary.tamu.edu","name":"National Archives and Records Administration","redirect":"https://www.bush41.org/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.bush41.org/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.bush41.org/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://bushlibrary.tamu.edu/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"bobber.info","name":"U.S. Army Corps of Engineers","redirect":"http://www.bobber.info/robots.txt","sitemap":"http://www.bobber.info/v20160617/ecards/pills/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://bobber.info/robots.txt","valid":true,"allowed":true,"sitemap":"http://www.bobber.info/v20160617/ecards/pills/sitemap.xml"}]},{"status":200,"url":"bie.edu","name":"Department of the Interior","redirect":"https://www.bie.edu/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.bie.edu/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"benefeds.com","name":"Office of Personnel Management","redirect":"https://www.benefeds.gov/robots.txt","sitemap":"https://www.benefeds.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.benefeds.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.benefeds.com/sitemap.xml"}]},{"status":500,"url":"badgerlandcfc.org","name":"Combined Federal Campaign","redirect":"http://www.badgerlandcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"aww.aww-sp.com","name":"U.S. Coast Guard","redirect":"http://aww.aww-sp.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://aww.aww-sp.com/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"auxpa.org","name":"U.S. Coast Guard Auxiliary","redirect":"http://auxpa.org/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"http://auxpa.org/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.auxpa.org","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"http://www.auxpa.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1734148194754,"status":404,"redirect":"http://wow.uscgaux.info/content.php?unit=a-dept","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"atlanticcoastcfc.org","name":"Combined Federal Campaign","redirect":"http://www.atlanticcoastcfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"askjan.org","name":"Department of Labor","redirect":"https://askjan.org/robots.txt","sitemap":"https://askjan.org/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://askjan.org/robots.txt","valid":true,"allowed":true,"sitemap":"https://askjan.org/sitemap.xml"}]},{"status":404,"url":"armyfuturescommand.com","name":"Department of the Army","redirect":"http://www.armyfuturescommand.com/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://armyfuturescommand.com/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"armedforcessports.com","name":"Department of Defense","redirect":"https://armedforcessports.defense.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://armedforcessports.defense.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://armedforcessports.com/DesktopModules/SiteData/SiteMap.ashx","source":"robots.txt"},{"time":1734148194754,"status":200,"redirect":"https://armedforcessports.defense.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1737430190371,"status":302,"redirect":"http://www.armedforcessports.com/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://armedforcessports.defense.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1740965653672,"status":302,"redirect":"http://www.armedforcessports.com/","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":200,"redirect":"https://armedforcessports.defense.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":500,"redirect":"http://www.armedforcessports.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"amver.com","name":"U.S. Coast Guard","redirect":"https://amver.com/Error/Index","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://amver.com/Error/Index","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://amver.com/Error/Index","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":500,"redirect":"http://amver.com/Error/Index","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"amtrak.com","name":"National Railroad Passenger Corporation","redirect":"https://www.amtrak.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.amtrak.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.amtrak.com/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.amtrak.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.amtrak.com/sitemap.xml","source":"robots.txt"}]},{"status":404,"url":"americorpsconnect.org","name":"Corporation for National and Community Service","redirect":"http://www.americorpsconnect.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"alhurra.com","name":"U.S. Agency for Global Media","redirect":"https://www.alhurra.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.alhurra.com/robots.txt","valid":true,"allowed":false,"sitemap":null}]},{"status":500,"url":"alaskacfc.org","name":"Combined Federal Campaign","redirect":"http://www.alaskacfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":202,"url":"ajb.org","name":"Employment and Training Administration","redirect":"https://www.careeronestop.org/JobSearch/FindJobs/state-job-banks.aspx","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":403,"redirect":"https://www.careeronestop.org/JobSearch/FindJobs/state-job-banks.aspx","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.ajb.org/","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":403,"redirect":"https://www.careeronestop.org/JobSearch/FindJobs/state-job-banks.aspx","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":202,"redirect":"https://www.careeronestop.org/JobSearch/FindJobs/state-job-banks.aspx","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.ajb.org/","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"airspacemag.com","name":"Smithsonian Institution","redirect":"https://www.smithsonianmag.com/category/air-space-magazine/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.smithsonianmag.com/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.smithsonianmag.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.smithsonianmag.com/sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"airforce.com","name":"Department of the Air Force","redirect":"https://www.airforce.com/robots.txt","sitemap":"https://www.airforce.com/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://airforce.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.airforce.com/sitemap.xml"}]},{"status":200,"url":"airdefenseartillery.com","name":"Department of Defense","redirect":"https://airdefenseartillery.com/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://airdefenseartillery.com/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"agribank.com","name":"Farm Credit Administration","redirect":"https://www.agribank.com/robots.txt","sitemap":"https://www.agribank.com/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.agribank.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.agribank.com/sitemap_index.xml"}]},{"status":200,"url":"agfirst.com","name":"Farm Credit Administration","redirect":"https://www.agfirst.com/robots.txt","sitemap":"https://www.agfirst.com/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.agfirst.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.agfirst.com/sitemap.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"afthunderbirds.com","name":"Department of the Air Force","redirect":"https://afthunderbirds.com/robots.txt","sitemap":"https://afthunderbirds.com/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"http://afthunderbirds.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://afthunderbirds.com/sitemap_index.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.afthunderbirds.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"afreserve.com","name":"Department of the Air Force","redirect":"https://www.airforce.com/afreserve/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.airforce.com/afreserve/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.afreserve.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"afpmb.org","name":"Department of Defense","redirect":"http://www.afpmb.org/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://afpmb.org/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":500,"url":"afnpacific.net","name":"Department of Defense","redirect":"http://afnpacific.net/robots.txt","sitemap":null,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.afnpacific.net","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://pacific.afn.mil/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"meta","allowed":true,"status":400,"url":"afneurope.net","name":"Department of Defense","redirect":"http://afneurope.net/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":400,"redirect":"http://afneurope.net/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.afneurope.net","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://europe.afn.mil/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1734148194754,"status":500,"redirect":"http://afneurope.net/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"afacademy.af.edu","name":"Department of Defense","redirect":"http://www.afacademy.af.edu/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"adlnet.org","name":"Department of Defense","redirect":"https://www.hoverwatch.com/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.adlnet.org/robots.txt/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.adlnet.org","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":200,"redirect":"https://www.adlnet.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":403,"url":"academyadmissions.com","name":"Department of Defense","redirect":"https://www.academyadmissions.com/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.academyadmissions.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.academyadmissions.com/sitemap_index.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.academyadmissions.com","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"aal.army","name":"Department of the Army","redirect":"https://aal.army/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://aal.army/robots.txt","valid":true,"allowed":true,"sitemap":"https://aal.army/sitemap.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.aal.army","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://aal.army/robots.txt","valid":true,"allowed":true,"sitemap":"https://aal.army/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"3riverscfc.org","name":"Combined Federal Campaign","redirect":"http://www.3riverscfc.org/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"18f.us","name":"General Services Administration","redirect":"https://18f.us/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://18f.us/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.18f.us","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":200,"redirect":"https://18f.us/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":500,"redirect":"http://www.18f.us/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"data.medicaid.gov","name":"Department of Health and Human Services","redirect":"https://data.medicaid.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://data.medicaid.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.data.medicaid.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"openpaymentsdata.cms.gov","name":"Department of Health and Human Services","redirect":"https://openpaymentsdata.cms.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://openpaymentsdata.cms.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.openpaymentsdata.cms.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"whdpc.gov","name":"Western Hemisphere Drug Policy Commission","redirect":"http://www.whdpc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.whdpc.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"senate.gov","name":"United States Senate","redirect":"https://www.senate.gov/pagelayout/general/one_item_and_teasers/file_not_found.htm","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.senate.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.senate.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"sen.gov","name":"United States Senate","redirect":"https://sen.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://sen.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.sen.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"uscc.gov","name":"U.S.-China Economic and Security Review Commission","redirect":"https://www.uscc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.uscc.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ussemiquincentennial.gov","name":"U.S. Semiquincentennial Commission","redirect":"https://ussemiquincentennial.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://ussemiquincentennial.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.ussemiquincentennial.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"usa250.gov","name":"U.S. Semiquincentennial Commission","redirect":"https://america250.org/robots.txt","sitemap":"https://america250.org/sitemap_index.xml","history":[{"time":1726617445612,"status":404,"redirect":"http://usa250.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.usa250.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"america250.gov","name":"U.S. Semiquincentennial Commission","redirect":"https://america250.org/robots.txt","sitemap":"https://america250.org/sitemap_index.xml","history":[{"time":1726617445612,"status":404,"redirect":"http://america250.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.america250.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"uscp.gov","name":"U.S. Capitol Police","redirect":"http://www.uscp.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.uscp.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"uscapitolpolice.gov","name":"U.S. Capitol Police","redirect":"http://www.uscapitolpolice.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ushr.gov","name":"The Legislative Branch","redirect":"http://www.ushr.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ushouse.gov","name":"The Legislative Branch","redirect":"http://www.ushouse.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"tmdbhouse.gov","name":"The Legislative Branch","redirect":"http://www.tmdbhouse.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"taxreform.gov","name":"The Legislative Branch","redirect":"http://www.taxreform.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"speaker.gov","name":"The Legislative Branch","redirect":"https://www.speaker.gov/robots.txt","sitemap":"https://www.speaker.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.speaker.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.speaker.gov/sitemap.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.speaker.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"republicanwhip.gov","name":"The Legislative Branch","redirect":"https://www.majoritywhip.gov/components/util/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.majoritywhip.gov/components/util/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.republicanwhip.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"republicans.gov","name":"The Legislative Branch","redirect":"http://www.republicans.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"republicanleader.gov","name":"The Legislative Branch","redirect":"https://www.majorityleader.gov/components/util/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.majorityleader.gov/components/util/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.republicanleader.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ppdcecc.gov","name":"The Legislative Branch","redirect":"https://www.ppdcecc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.ppdcecc.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"minoritywhip.gov","name":"The Legislative Branch","redirect":"https://www.minoritywhip.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":403,"redirect":"https://www.minoritywhip.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.minoritywhip.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"majoritywhip.gov","name":"The Legislative Branch","redirect":"https://www.majoritywhip.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.majoritywhip.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.majoritywhip.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"majorityleader.gov","name":"The Legislative Branch","redirect":"https://www.majorityleader.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.majorityleader.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.majorityleader.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"jct.gov","name":"The Legislative Branch","redirect":"https://www.jct.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.jct.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.jct.gov","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://www.jct.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.jct.gov/sitemap.xml/","source":"robots.txt"}]},{"status":500,"url":"housenewsletters.gov","name":"The Legislative Branch","redirect":"http://www.housenewsletters.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"houselive.gov","name":"The Legislative Branch","redirect":"http://www.houselive.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"housedems.gov","name":"The Legislative Branch","redirect":"https://www.housedems.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"housedemocrats.gov","name":"The Legislative Branch","redirect":"https://www.housedemocrats.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"housed.gov","name":"The Legislative Branch","redirect":"http://www.housed.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"housecommunications.gov","name":"The Legislative Branch","redirect":"https://www.housecommunications.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.housecommunications.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.housecommunications.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"house.gov","name":"The Legislative Branch","redirect":"https://www.house.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":403,"redirect":"https://www.house.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.house.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":403,"redirect":"http://www.house.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"gopwhip.gov","name":"The Legislative Branch","redirect":"https://www.majoritywhip.gov/components/util/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.majoritywhip.gov/components/util/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.gopwhip.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"gopleader.gov","name":"The Legislative Branch","redirect":"https://www.majorityleader.gov/components/util/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.majorityleader.gov/components/util/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.gopleader.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"gop.gov","name":"The Legislative Branch","redirect":"https://www.gop.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.gop.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.gop.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":410,"url":"dems.gov","name":"The Legislative Branch","redirect":"https://www.dems.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":410,"redirect":"https://www.dems.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.dems.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"democrats.gov","name":"The Legislative Branch","redirect":"http://www.democrats.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"democraticwhip.gov","name":"The Legislative Branch","redirect":"https://www.democraticwhip.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":403,"redirect":"https://www.democraticwhip.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.democraticwhip.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"democraticleader.gov","name":"The Legislative Branch","redirect":"https://www.democraticleader.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":403,"redirect":"https://www.democraticleader.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.democraticleader.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"csce.gov","name":"The Legislative Branch","redirect":"https://www.csce.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.csce.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.csce.gov/wp-sitemap.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.csce.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"chinacommission.gov","name":"The Legislative Branch","redirect":"http://www.chinacommission.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"china-commission.gov","name":"The Legislative Branch","redirect":"http://www.china-commission.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"cecc.gov","name":"The Legislative Branch","redirect":"http://www.cecc.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cecc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.cecc.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"assistantdemocraticleader.gov","name":"The Legislative Branch","redirect":"http://www.assistantdemocraticleader.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"stennis.gov","name":"Stennis Center for Public Service","redirect":"https://stennis.gov/robots.txt","sitemap":"https://stennis.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://stennis.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://stennis.gov/sitemap_index.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.stennis.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"openworld.gov","name":"Open World Leadership Center","redirect":"https://www.openworld.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.openworld.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.openworld.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":400,"url":"coil.gov","name":"Open World Leadership Center","redirect":"http://www.coil.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.coil.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ocwr.gov","name":"Office of Congressional Workplace Rights","redirect":"https://www.ocwr.gov/robots.txt","sitemap":"https://www.ocwr.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ocwr.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ocwr.gov/sitemap_index.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.ocwr.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"compliance.gov","name":"Office of Congressional Workplace Rights","redirect":"http://www.compliance.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://compliance.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.compliance.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"medpac.gov","name":"Medical Payment Advisory Commission","redirect":"https://www.medpac.gov/robots.txt","sitemap":"https://www.medpac.gov/wp-sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.medpac.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.medpac.gov/wp-sitemap.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.medpac.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"macpac.gov","name":"Medicaid and CHIP Payment and Access Commission","redirect":"https://www.macpac.gov/robots.txt","sitemap":"https://www.macpac.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"http://macpac.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.macpac.gov/sitemap.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.macpac.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"womenshistorymonth.gov","name":"Library of Congress","redirect":"https://womenshistorymonth.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://womenshistorymonth.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"uscongress.gov","name":"Library of Congress","redirect":"https://www.congress.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"unitedstatescongress.gov","name":"Library of Congress","redirect":"https://www.congress.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"tps.gov","name":"Library of Congress","redirect":"https://www.loc.gov/programs/teachers/about-this-program/teaching-with-primary-sources-partner-program/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.loc.gov/programs/teachers/about-this-program/teaching-with-primary-sources-partner-program/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.loc.gov/programs/teachers/about-this-program/teaching-with-primary-sources-partner-program/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.tps.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"thomas.gov","name":"Library of Congress","redirect":"http://www.thomas.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"section108.gov","name":"Library of Congress","redirect":"https://section108.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://section108.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":404,"redirect":"https://section108.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":500,"redirect":"https://section108.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"read.gov","name":"Library of Congress","redirect":"https://read.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://read.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"nativeamericanheritagemonth.gov","name":"Library of Congress","redirect":"https://nativeamericanheritagemonth.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://nativeamericanheritagemonth.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1734148194754,"status":404,"redirect":"https://nativeamericanheritagemonth.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"loctps.gov","name":"Library of Congress","redirect":"https://www.loc.gov/programs/teachers/about-this-program/teaching-with-primary-sources-partner-program/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.loc.gov/programs/teachers/about-this-program/teaching-with-primary-sources-partner-program/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.loc.gov/programs/teachers/about-this-program/teaching-with-primary-sources-partner-program/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.loctps.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"loc.gov","name":"Library of Congress","redirect":"https://www.loc.gov/robots.txt","sitemap":"https://www.loc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://loc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.loc.gov/sitemap.xml"}]},{"status":200,"url":"literacy.gov","name":"Library of Congress","redirect":"https://read.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://read.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://read.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.literacy.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"lis.gov","name":"Library of Congress","redirect":"http://www.lis.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"libraryofcongress.gov","name":"Library of Congress","redirect":"https://www.loc.gov/robots.txt","sitemap":"https://www.loc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.loc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.loc.gov/sitemap.xml"}]},{"status":500,"url":"lctl.gov","name":"Library of Congress","redirect":"http://www.lctl.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"law.gov","name":"Library of Congress","redirect":"https://www.loc.gov/research-centers/law-library-of-congress/about-this-research-center/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.loc.gov/research-centers/law-library-of-congress/about-this-research-center/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.loc.gov/research-centers/law-library-of-congress/about-this-research-center/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.law.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"jewishheritagemonth.gov","name":"Library of Congress","redirect":"https://jewishheritagemonth.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://jewishheritagemonth.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"jewishheritage.gov","name":"Library of Congress","redirect":"http://www.jewishheritagemonth.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"hispanicheritagemonth.gov","name":"Library of Congress","redirect":"https://hispanicheritagemonth.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://hispanicheritagemonth.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"digitizationguidelines.gov","name":"Library of Congress","redirect":"https://www.digitizationguidelines.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.digitizationguidelines.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"digitalpreservation.gov","name":"Library of Congress","redirect":"https://www.digitalpreservation.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://digitalpreservation.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"currencyreaderapplication.gov","name":"Library of Congress","redirect":"https://currencyreader.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://currencyreader.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://currencyreader.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://currencyreaderapplication.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"currencyreader.gov","name":"Library of Congress","redirect":"https://currencyreader.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://currencyreader.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"crs.gov","name":"Library of Congress","redirect":"http://www.crs.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"crb.gov","name":"Library of Congress","redirect":"https://www.crb.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://crb.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":404,"redirect":"https://www.crb.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":500,"redirect":"https://www.crb.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"copyrightclaimsboard.gov","name":"Library of Congress","redirect":"http://www.copyrightclaimsboard.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"copyright.gov","name":"Library of Congress","redirect":"https://copyright.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://copyright.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"congress.gov","name":"Library of Congress","redirect":"https://www.congress.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1729037740707,"status":403,"redirect":"https://www.congress.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":500,"redirect":"http://www.congress.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ccb.gov","name":"Library of Congress","redirect":"https://ccb.gov/robots.txt","sitemap":"https://ccb.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://ccb.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://ccb.gov/sitemap.xml"}]},{"status":404,"url":"blackhistorymonth.gov","name":"Library of Congress","redirect":"https://blackhistorymonth.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://blackhistorymonth.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"asianpacificheritage.gov","name":"Library of Congress","redirect":"https://asianpacificheritage.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://asianpacificheritage.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"americaslibrary.gov","name":"Library of Congress","redirect":"https://www.loc.gov/families/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.loc.gov/families/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.loc.gov/families/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://americaslibrary.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"americanmemory.gov","name":"Library of Congress","redirect":"https://www.loc.gov/robots.txt","sitemap":"https://www.loc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.loc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.loc.gov/sitemap.xml"}]},{"status":200,"url":"africanamericanhistorymonth.gov","name":"Library of Congress","redirect":"https://blackhistorymonth.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://blackhistorymonth.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://blackhistorymonth.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://www.africanamericanhistorymonth.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"usgovernmentmanual.gov","name":"Government Publishing Office","redirect":"https://usgovernmentmanual.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://usgovernmentmanual.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"uscode.gov","name":"Government Publishing Office","redirect":"https://www.govinfo.gov/app/collection/uscode","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.govinfo.gov/app/collection/uscode","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.govinfo.gov/app/collection/uscode","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.uscode.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"senatecalendar.gov","name":"Government Publishing Office","redirect":"https://www.govinfo.gov/app/collection/ccal","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.govinfo.gov/app/collection/ccal","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.govinfo.gov/app/collection/ccal","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://senatecalendar.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"presidentialdocuments.gov","name":"Government Publishing Office","redirect":"https://www.govinfo.gov/app/collection/cpd","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.govinfo.gov/app/collection/cpd","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.govinfo.gov/app/collection/cpd","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://presidentialdocuments.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"ofr.gov","name":"Government Publishing Office","redirect":"https://www.archives.gov/files/federal-register/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.archives.gov/files/federal-register/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":500,"redirect":"http://www.ofr.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"housecalendar.gov","name":"Government Publishing Office","redirect":"https://www.govinfo.gov/app/collection/ccal","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.govinfo.gov/app/collection/ccal","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.govinfo.gov/app/collection/ccal","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://housecalendar.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"gpodev.gov","name":"Government Publishing Office","redirect":"http://www.gpodev.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"gpo.gov","name":"Government Publishing Office","redirect":"https://www.gpo.gov/robots.txt","sitemap":"https://www.gpo.gov/sitemap/sitemap-index.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.gpo.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.gpo.gov/sitemap/sitemap-index.xml"}]},{"status":200,"url":"govinfo.gov","name":"Government Publishing Office","redirect":"https://www.govinfo.gov/robots.txt","sitemap":"https://www.govinfo.gov/sitemap/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.govinfo.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.govinfo.gov/sitemap/sitemap.xml"}]},{"status":500,"url":"fedreg.gov","name":"Government Publishing Office","redirect":"http://www.fedreg.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":302,"url":"federalregister.gov","name":"Government Publishing Office","redirect":"https://unblock.federalregister.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://unblock.federalregister.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://unblock.federalregister.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://federalregister.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":200,"redirect":"https://unblock.federalregister.gov/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fdsys.gov","name":"Government Publishing Office","redirect":"https://www.govinfo.gov/about#fdsys-transition","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.govinfo.gov/about#fdsys-transition","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.govinfo.gov/about#fdsys-transition","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://fdsys.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fdlp.gov","name":"Government Publishing Office","redirect":"https://fdlp.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://fdlp.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":302,"url":"ecfr.gov","name":"Government Publishing Office","redirect":"https://unblock.federalregister.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://unblock.federalregister.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://unblock.federalregister.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://www.ecfr.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":200,"redirect":"https://unblock.federalregister.gov/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"congressionalrecord.gov","name":"Government Publishing Office","redirect":"https://www.govinfo.gov/app/collection/crec","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.govinfo.gov/app/collection/crec","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.govinfo.gov/app/collection/crec","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://congressionalrecord.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"congressionaldirectory.gov","name":"Government Publishing Office","redirect":"https://www.govinfo.gov/app/collection/cdir","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.govinfo.gov/app/collection/cdir","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.govinfo.gov/app/collection/cdir","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://congressionaldirectory.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"gaonet.gov","name":"Government Accountability Office","redirect":"http://www.gaonet.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"gaoinnovations.gov","name":"Government Accountability Office","redirect":"https://gaoinnovations.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://gaoinnovations.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"gaoinnovationlab.gov","name":"Government Accountability Office","redirect":"http://www.gaoinnovationlab.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"gaoinnovation.gov","name":"Government Accountability Office","redirect":"http://www.gaoinnovation.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"gao.gov","name":"Government Accountability Office","redirect":"https://www.gao.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.gao.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"fasab.gov","name":"Government Accountability Office","redirect":"https://fasab.gov/robots.txt","sitemap":"https://fasab.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://fasab.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://fasab.gov/sitemap.xml"}]},{"status":500,"url":"eseclab.gov","name":"Government Accountability Office","redirect":"http://www.eseclab.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"solarium.gov","name":"Cyberspace Solarium Commission","redirect":"https://www.solarium.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.solarium.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"cbonews.gov","name":"Congressional Budget Office","redirect":"http://www.cbonews.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://www.cbonews.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"cbo.gov","name":"Congressional Budget Office","redirect":"https://www.cbo.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cbo.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"visitthecapitol.gov","name":"Architect of the Capitol","redirect":"https://www.visitthecapitol.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.visitthecapitol.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"visitthecapital.gov","name":"Architect of the Capitol","redirect":"http://www.visitthecapital.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"http://www.visitthecapital.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"uscapitolgiftstore.gov","name":"Architect of the Capitol","redirect":"http://www.uscapitolgiftstore.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"uscapitolgiftshop.gov","name":"Architect of the Capitol","redirect":"http://www.uscapitolgiftshop.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://uscapitolgiftshop.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"uscapitol.gov","name":"Architect of the Capitol","redirect":"http://www.uscapitol.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"http://www.uscapitol.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"uscapitalgiftshop.gov","name":"Architect of the Capitol","redirect":"http://www.uscapitalgiftshop.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"uscapital.gov","name":"Architect of the Capitol","redirect":"http://www.uscapital.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"usbg.gov","name":"Architect of the Capitol","redirect":"https://www.usbg.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usbg.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"capitolgiftshop.gov","name":"Architect of the Capitol","redirect":"http://www.capitolgiftshop.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"capitol.gov","name":"Architect of the Capitol","redirect":"https://www.aoc.gov/virtual-capitol","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.aoc.gov/virtual-capitol","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.aoc.gov/virtual-capitol","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://capitol.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"capitalgiftshop.gov","name":"Architect of the Capitol","redirect":"http://www.capitalgiftshop.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"capital.gov","name":"Architect of the Capitol","redirect":"https://www.aoc.gov/virtual-capitol","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.aoc.gov/virtual-capitol","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.aoc.gov/virtual-capitol","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://capital.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"aoc.gov","name":"Architect of the Capitol","redirect":"https://www.aoc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.aoc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"ustaxcourt.gov","name":"U.S. Courts","redirect":"https://ustaxcourt.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://ustaxcourt.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"ussc.gov","name":"U.S. Courts","redirect":"http://www.ussc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"usprobation.gov","name":"U.S. Courts","redirect":"https://www.uscourts.gov/about-federal-courts/probation-and-pretrial-services","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.uscourts.gov/services-forms/probation-and-pretrial-services","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.usprobation.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"uscourts.gov","name":"U.S. Courts","redirect":"https://www.uscourts.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.uscourts.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"uscavc.gov","name":"U.S. Courts","redirect":"http://www.uscavc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"usc.gov","name":"U.S. Courts","redirect":"http://www.usc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"usbankruptcy.gov","name":"U.S. Courts","redirect":"https://www.uscourts.gov/court-programs/bankruptcy","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.uscourts.gov/services-forms/bankruptcy","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.usbankruptcy.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"pacer.gov","name":"U.S. Courts","redirect":"https://pacer.uscourts.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://pacer.uscourts.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"judicialconference.gov","name":"U.S. Courts","redirect":"https://www.uscourts.gov/administration-policies/governance-judicial-conference","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.uscourts.gov/about-federal-courts/governance-judicial-conference","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.judicialconference.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fjc.gov","name":"U.S. Courts","redirect":"https://www.fjc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fjc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"federalrules.gov","name":"U.S. Courts","redirect":"https://www.uscourts.gov/forms-rules","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.uscourts.gov/rules-policies","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.federalrules.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"federalprobation.gov","name":"U.S. Courts","redirect":"https://www.uscourts.gov/about-federal-courts/probation-and-pretrial-services","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.uscourts.gov/services-forms/probation-and-pretrial-services","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.federalprobation.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"federalcourts.gov","name":"U.S. Courts","redirect":"https://www.uscourts.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.uscourts.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.federalcourts.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"cavc.gov","name":"U.S. Courts","redirect":"http://www.cavc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"bankruptcy.gov","name":"U.S. Courts","redirect":"https://www.uscourts.gov/court-programs/bankruptcy","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.uscourts.gov/services-forms/bankruptcy","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"http://www.bankruptcy.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"supremecourtus.gov","name":"The Supreme Court","redirect":"https://www.supremecourt.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"supremecourt.gov","name":"The Supreme Court","redirect":"https://www.supremecourt.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.supremecourt.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"supreme-court.gov","name":"The Supreme Court","redirect":"http://www.supreme-court.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"scuspd.gov","name":"The Supreme Court","redirect":"https://www.scuspd.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.scuspd.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"scus.gov","name":"The Supreme Court","redirect":"http://www.scus.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"scinet.gov","name":"The Supreme Court","redirect":"http://www.scinet.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"scinet-test.gov","name":"The Supreme Court","redirect":"http://www.scinet-test.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"sc-us.gov","name":"The Supreme Court","redirect":"http://www.sc-us.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"fd.gov","name":"Administrative Office of the U. S. Courts","redirect":"http://www.fd.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"rpo.gov","name":"gov Administration","redirect":"http://www.rpo.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pubservices.gov","name":"gov Administration","redirect":"http://www.pubservices.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"psup.gov","name":"gov Administration","redirect":"http://www.psup.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"psd.gov","name":"gov Administration","redirect":"http://www.psd.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"owc.gov","name":"gov Administration","redirect":"http://www.owc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"osdls.gov","name":"gov Administration","redirect":"http://www.osdls.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nrpo.gov","name":"gov Administration","redirect":"http://www.nrpo.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"jpo.gov","name":"gov Administration","redirect":"http://www.jpo.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"fraho.gov","name":"gov Administration","redirect":"http://www.fraho.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"erpo.gov","name":"gov Administration","redirect":"http://www.erpo.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ecfc.gov","name":"gov Administration","redirect":"http://www.ecfc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"domainops.gov","name":"gov Administration","redirect":"http://www.domainops.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ata.gov","name":"gov Administration","redirect":"http://www.ata.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ustda.gov","name":"United States Trade and Development Agency","redirect":"https://www.ustda.gov/robots.txt","sitemap":"https://www.ustda.gov/wp-sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ustda.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ustda.gov/wp-sitemap.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.ustda.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.ustda.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ustda.gov/wp-sitemap.xml","source":"robots.txt"},{"time":1734148194754,"status":500,"redirect":"http://www.ustda.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"uspsoig.gov","name":"United States Postal Service","redirect":"https://www.uspsoig.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.uspsoig.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.uspsoig.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"uspsinnovates.gov","name":"United States Postal Service","redirect":"http://www.uspsinnovates.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"uspsinformeddelivery.gov","name":"United States Postal Service","redirect":"http://www.uspsinformeddelivery.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"usps.gov","name":"United States Postal Service","redirect":"https://www.usps.com/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.usps.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"http://www.usps.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":200,"redirect":"https://www.usps.com/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":500,"redirect":"https://www.usps.com/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"uspis.gov","name":"United States Postal Service","redirect":"https://www.uspis.gov/robots.txt","sitemap":"https://www.uspis.gov/sitemap_index.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.uspis.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.uspis.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"purchasing.gov","name":"United States Postal Service","redirect":"http://www.purchasing.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"postoffice.gov","name":"United States Postal Service","redirect":"http://www.postoffice.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"myusps.gov","name":"United States Postal Service","redirect":"http://www.myusps.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"mail.gov","name":"United States Postal Service","redirect":"http://www.mail.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"changeofaddress.gov","name":"United States Postal Service","redirect":"http://www.changeofaddress.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"usitcoig.gov","name":"United States International Trade Commission","redirect":"http://www.usitcoig.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"usitc.gov","name":"United States International Trade Commission","redirect":"https://www.usitc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usitc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.usitc.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"usidfc.gov","name":"United States International Development Finance Corporation","redirect":"https://www.dfc.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dfc.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"https://www.usidfc.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://usidfc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"usdfc.gov","name":"United States International Development Finance Corporation","redirect":"https://www.dfc.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dfc.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"https://www.usdfc.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://usdfc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"opic.gov","name":"United States International Development Finance Corporation","redirect":"http://www.opic.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"idfc.gov","name":"United States International Development Finance Corporation","redirect":"https://www.dfc.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dfc.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.idfc.gov","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":301,"redirect":"https://idfc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":500,"redirect":"http://www.idfc.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"dfc.gov","name":"United States International Development Finance Corporation","redirect":"https://www.dfc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.dfc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.dfc.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"usich.gov","name":"United States Interagency Council on Homelessness","redirect":"https://usich.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://usich.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.usich.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"ich.gov","name":"United States Interagency Council on Homelessness","redirect":"https://www.ich.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"usip.gov","name":"United States Institute of Peace","redirect":"https://usip.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://usip.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"ushmm.gov","name":"United States Holocaust Memorial Museum","redirect":"http://www.ushmm.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"usgcrp.gov","name":"United States Global Change Research Program","redirect":"http://www.usgcrp.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"ipcc-wg3.gov","name":"United States Global Change Research Program","redirect":"https://www.ipcc-wg3.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"http://ipcc-wg3.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"globalchange.gov","name":"United States Global Change Research Program","redirect":"http://www.globalchange.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"voa.gov","name":"United States Agency for Global Media","redirect":"https://www.voanews.com/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.voanews.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.voanews.com/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.voanews.com/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.voanews.com/sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"usagm.gov","name":"United States Agency for Global Media","redirect":"https://www.usagm.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usagm.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usagm.gov/wp-sitemap.xml"},{"time":1739509031548,"status":200,"redirect":"https://www.usagm.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usagm.gov/wp-sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"ocb.gov","name":"United States Agency for Global Media","redirect":"http://www.ocb.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ibb.gov","name":"United States Agency for Global Media","redirect":"https://www.usagm.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usagm.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usagm.gov/wp-sitemap.xml"},{"time":1739509031548,"status":200,"redirect":"https://www.usagm.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usagm.gov/wp-sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"bbg.gov","name":"United States Agency for Global Media","redirect":"https://www.usagm.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usagm.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usagm.gov/wp-sitemap.xml"},{"time":1739509031548,"status":200,"redirect":"https://www.usagm.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usagm.gov/wp-sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"usadf.gov","name":"United States African Development Foundation","redirect":"https://usadf.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"https://usadf.gov/robots.txt/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"https://usadf.gov/robots.txt/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"adf.gov","name":"United States African Development Foundation","redirect":"https://www.adf.gov//","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"access-board.gov","name":"United States Access Board","redirect":"https://www.access-board.gov/robots.txt","sitemap":"https://www.access-board.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"jwod.gov","name":"United States AbilityOne","redirect":"http://www.jwod.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"abilityone.gov","name":"United States AbilityOne","redirect":"https://abilityone.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1734148194754,"status":500,"redirect":"http://www.abilityone.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"peacecorpsoig.gov","name":"U.S. Peace Corps","redirect":"https://www.peacecorpsoig.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.peacecorpsoig.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://www.peacecorpsoig.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":403,"url":"peacecorps.gov","name":"U.S. Peace Corps","redirect":"https://www.peacecorps.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.peacecorps.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.peacecorps.gov/sitemap.xml"}]},{"status":500,"url":"oscnet.gov","name":"U.S. Office of Special Counsel","redirect":"http://www.oscnet.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"osc.gov","name":"U.S. Office of Special Counsel","redirect":"https://osc.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://osc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"nwtrb.gov","name":"U.S. Nuclear Waste Technical Review Board","redirect":"https://www.nwtrb.gov/robots.txt","sitemap":"http://www.nwtrb.gov/sitemap/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nwtrb.gov/robots.txt","valid":true,"allowed":true,"sitemap":"http://www.nwtrb.gov/sitemap/sitemap.xml"}]},{"status":200,"url":"wildfire.gov","name":"U.S. Department of Agriculture","redirect":"https://www.wildfire.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.wildfire.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"usdapii.gov","name":"U.S. Department of Agriculture","redirect":"http://www.usdapii.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"usda.gov","name":"U.S. Department of Agriculture","redirect":"https://www.usda.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usda.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"symbols.gov","name":"U.S. Department of Agriculture","redirect":"https://symbols.gov/robots.txt","sitemap":"https://symbols.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://symbols.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://symbols.gov/sitemap.xml"}]},{"status":200,"url":"rural.gov","name":"U.S. Department of Agriculture","redirect":"https://www.rural.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.rural.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"recreation.gov","name":"U.S. Department of Agriculture","redirect":"https://www.recreation.gov/robots.txt","sitemap":"https://www.recreation.gov/sitemap.xml.gz","history":[{"time":1726617445612,"status":200,"redirect":"https://www.recreation.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.recreation.gov/sitemap.xml.gz"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"rec.gov","name":"U.S. Department of Agriculture","redirect":"https://www.recreation.gov/robots.txt","sitemap":"https://www.recreation.gov/sitemap.xml.gz","history":[{"time":1726617445612,"status":200,"redirect":"https://www.recreation.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.recreation.gov/sitemap.xml.gz"}]},{"status":200,"url":"nwcg.gov","name":"U.S. Department of Agriculture","redirect":"https://www.nwcg.gov/robots.txt","sitemap":"https://www.nwcg.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nwcg.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"nutrition.gov","name":"U.S. Department of Agriculture","redirect":"https://www.nutrition.gov/robots.txt","sitemap":"https://nutrition.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1739509031548,"status":500,"redirect":"https://www.nutrition.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":500,"redirect":"https://www.nutrition.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"nafri.gov","name":"U.S. Department of Agriculture","redirect":"https://www.nafri.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://nafri.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1742146243896,"status":200,"redirect":"https://nafri.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"myplate.gov","name":"U.S. Department of Agriculture","redirect":"https://www.myplate.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.myplate.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"lcacommons.gov","name":"U.S. Department of Agriculture","redirect":"https://www.lcacommons.gov/robots.txt","sitemap":"https://lcacommons.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1739509031548,"status":500,"redirect":"https://www.lcacommons.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":500,"redirect":"https://www.lcacommons.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":200,"redirect":"https://www.lcacommons.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://lcacommons.gov/sitemap.xml","source":"robots.txt"},{"time":1743826667324,"status":500,"redirect":"https://www.lcacommons.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"itap.gov","name":"U.S. Department of Agriculture","redirect":"https://www.itap.gov/robots.txt","sitemap":"https://itap.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1739509031548,"status":500,"redirect":"https://www.itap.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":500,"redirect":"https://www.itap.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"invasivespeciesinfo.gov","name":"U.S. Department of Agriculture","redirect":"https://www.invasivespeciesinfo.gov/robots.txt","sitemap":"https://www.invasivespeciesinfo.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1739509031548,"status":500,"redirect":"https://www.invasivespeciesinfo.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":500,"redirect":"https://www.invasivespeciesinfo.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"forestsandrangelands.gov","name":"U.S. Department of Agriculture","redirect":"https://forestsandrangelands.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://forestsandrangelands.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"fedsfeedfamilies.gov","name":"U.S. Department of Agriculture","redirect":"http://www.fedsfeedfamilies.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"farmers.gov","name":"U.S. Department of Agriculture","redirect":"https://www.farmers.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.farmers.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"farmerfairness.gov","name":"U.S. Department of Agriculture","redirect":"https://www.usda.gov/farmerfairness","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usda.gov/farmerfairness","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usda.gov/farmerfairness","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://farmerfairness.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"execsec.gov","name":"U.S. Department of Agriculture","redirect":"https://www.usda.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usda.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usda.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://execsec.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"empowhr.gov","name":"U.S. Department of Agriculture","redirect":"http://www.empowhr.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"dietaryguidelines.gov","name":"U.S. Department of Agriculture","redirect":"https://www.dietaryguidelines.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.dietaryguidelines.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"connect.gov","name":"U.S. Department of Agriculture","redirect":"https://www.connect.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"choosemyplate.gov","name":"U.S. Department of Agriculture","redirect":"https://www.myplate.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.myplate.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"bosque.gov","name":"U.S. Department of Agriculture","redirect":"https://bosque.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://bosque.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"biopreferred.gov","name":"U.S. Department of Agriculture","redirect":"https://www.biopreferred.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"arsusda.gov","name":"U.S. Department of Agriculture","redirect":"http://www.arsusda.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"ars-grin.gov","name":"U.S. Department of Agriculture","redirect":"https://ars-grin.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://ars-grin.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"ag.gov","name":"U.S. Department of Agriculture","redirect":"http://www.ag.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"aff.gov","name":"U.S. Department of Agriculture","redirect":"https://www.aff.gov/robots.txt","sitemap":"https://www.aff.gov/aff-site-index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.aff.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.aff.gov/aff-site-index.xml"}]},{"status":500,"url":"22007apply.gov","name":"U.S. Department of Agriculture","redirect":"http://www.22007apply.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://22007apply.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://22007apply.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"https://www.usda.gov/about-usda/general-information/initiatives-and-highlighted-programs/22007","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":403,"redirect":"https://www.usda.gov/about-usda/general-information/initiatives-and-highlighted-programs/22007","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":530,"redirect":"https://www.22007apply.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"uscirf.gov","name":"U.S. Commission on International Religious Freedom","redirect":"https://www.uscirf.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.uscirf.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"usccr.gov","name":"U.S. Commission on Civil Rights","redirect":"https://www.usccr.gov/robots.txt","sitemap":"https://www.usccr.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usccr.gov//robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cssbmb.gov","name":"U.S. Commission on Civil Rights","redirect":"https://cssbmb.gov/robots.txt","sitemap":"https://cssbmb.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://cssbmb.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://cssbmb.gov/sitemap.xml"}]},{"status":500,"url":"civilrightsusa.gov","name":"U.S. Commission on Civil Rights","redirect":"http://www.civilrightsusa.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"cfa.gov","name":"U.S. Commission of Fine Arts","redirect":"https://cfa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cfa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"heritageabroad.gov","name":"U.S. Commission for the Preservation of Americas Heritage Abroad","redirect":"https://www.heritageabroad.gov/robots.txt","sitemap":"https://www.heritageabroad.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.heritageabroad.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.heritageabroad.gov/sitemap_index.xml"}]},{"status":500,"url":"workwithusaid.gov","name":"U.S. Agency for International Development","redirect":"https://www.workwithusaid.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://workwithusaid.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://workwithusaid.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":200,"redirect":"https://www.workwithusaid.gov/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"usaid.gov","name":"U.S. Agency for International Development","redirect":"https://www.usaid.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://usaid.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.usaid.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1740965653672,"status":200,"redirect":"https://www.usaid.gov/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"prosperafrica.gov","name":"U.S. Agency for International Development","redirect":"https://www.prosperafrica.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://prosperafrica.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://prosperafrica.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"pmi.gov","name":"U.S. Agency for International Development","redirect":"https://www.pmi.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":503,"redirect":"http://pmi.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.pmi.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.pmi.gov/sitemap_index.xml","source":"robots.txt"},{"time":1730174316792,"status":200,"redirect":"https://pmi.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"neglecteddiseases.gov","name":"U.S. Agency for International Development","redirect":"https://www.neglecteddiseases.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":503,"redirect":"http://neglecteddiseases.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.neglecteddiseases.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.neglecteddiseases.gov/sitemap_index.xml","source":"robots.txt"},{"time":1730174316792,"status":200,"redirect":"https://neglecteddiseases.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"foreignassistance.gov","name":"U.S. Agency for International Development","redirect":"https://foreignassistance.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"feedthefuture.gov","name":"U.S. Agency for International Development","redirect":"https://www.feedthefuture.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.feedthefuture.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.feedthefuture.gov/sitemap_index.xml"},{"time":1739509031548,"status":200,"redirect":"https://www.feedthefuture.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.feedthefuture.gov/sitemap_index.xml","source":"robots.txt"}]},{"status":200,"url":"dfafacts.gov","name":"U.S. Agency for International Development","redirect":"https://dfafacts.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.dfafacts.gov","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://dfafacts.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1737430190371,"status":404,"redirect":"https://dfafacts.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"childreninadversity.gov","name":"U.S. Agency for International Development","redirect":"https://www.childreninadversity.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":503,"redirect":"http://childreninadversity.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.childreninadversity.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.childreninadversity.gov/sitemap_index.xml","source":"robots.txt"},{"time":1730174316792,"status":200,"redirect":"https://childreninadversity.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":404,"url":"worldwar1centennial.gov","name":"The United States World War One Centennial Commission","redirect":"https://www.worldwar1centennial.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://worldwar1centennial.org/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://worldwar1centennial.org/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1740965653672,"status":404,"redirect":"http://www.worldwar1centennial.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":500,"redirect":"http://www.worldwar1centennial.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":400,"url":"ptf.gov","name":"The Intelligence Community","redirect":"http://www.ptf.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"tvaoig.gov","name":"Tennessee Valley Authority","redirect":"https://www.tvaoig.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.tvaoig.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":403,"url":"tva.gov","name":"Tennessee Valley Authority","redirect":"https://www.tva.com/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":403,"redirect":"https://www.tva.com/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":302,"redirect":"https://www.tva.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"stb.gov","name":"Surface Transportation Board","redirect":"https://www.stb.gov/robots.txt","sitemap":"https://www.stb.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.stb.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.stb.gov/sitemap_index.xml"}]},{"status":200,"url":"sji.gov","name":"State Justice Institute","redirect":"https://www.sji.gov/robots.txt","sitemap":"https://www.sji.gov/wp-sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.sji.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.sji.gov/wp-sitemap.xml"}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"sbrc.gov","name":"Southwest Border Regional Commission","redirect":"https://sbrc.gov/robots.txt","sitemap":null,"history":[{"time":1734148194754,"status":500,"redirect":"http://www.sbrc.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://sbrc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"scrc.gov","name":"Southeast Crescent Regional Commission","redirect":"https://scrc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://scrc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://scrc.gov/sitemap_index.xml"},{"time":1730174316792,"status":200,"redirect":"https://scrc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1734148194754,"status":404,"redirect":"https://www.scrc.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ssab.gov","name":"Social Security Advisory Board","redirect":"https://www.ssab.gov/robots.txt","sitemap":"https://www.ssab.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ssab.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ssab.gov/sitemap.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ssa.gov","name":"Social Security Administration","redirect":"https://www.ssa.gov/robots.txt","sitemap":"https://www.ssa.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ssa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ssa.gov/sitemap.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"socialsecurity.gov","name":"Social Security Administration","redirect":"https://www.ssa.gov/robots.txt","sitemap":"https://www.ssa.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ssa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ssa.gov/sitemap.xml"}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"segurosocial.gov","name":"Social Security Administration","redirect":"https://www.ssa.gov/espanol/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.ssa.gov/espanol/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"smithsonian.gov","name":"Smithsonian Institution","redirect":"http://www.smithsonian.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.si.edu/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.si.edu/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":404,"url":"itis.gov","name":"Smithsonian Institution","redirect":"https://www.itis.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"x-robots-tag","history":[{"time":1726617445612,"status":200,"redirect":"https://itis.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"https://www.itis.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://itis.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://itis.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"sbir.gov","name":"Small Business Administration","redirect":"https://www.sbir.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.sbir.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"sba.gov","name":"Small Business Administration","redirect":"https://www.sba.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.sba.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nwbc.gov","name":"Small Business Administration","redirect":"https://www.nwbc.gov/robots.txt","sitemap":"https://www.nwbc.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"business.gov","name":"Small Business Administration","redirect":"http://www.business.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"sss.gov","name":"Selective Service System","redirect":"https://www.sss.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.sss.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.sss.gov/sitemap.xml"},{"time":1743826667324,"status":200,"redirect":"https://www.sss.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.sss.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"sec.gov","name":"Securities and Exchange Commission","redirect":"https://www.sec.gov/robots.txt","sitemap":"https://www.sec.gov/sec-sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.sec.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.sec.gov/sec-sitemap.xml"}]},{"status":200,"url":"investor.gov","name":"Securities and Exchange Commission","redirect":"https://www.investor.gov/robots.txt","sitemap":"https://www.sec.gov/sec-sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.investor.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.sec.gov/sec-sitemap.xml"}]},{"status":200,"url":"rrb.gov","name":"Railroad Retirement Board","redirect":"https://rrb.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.rrb.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"pbrb.gov","name":"Public Buildings Reform Board","redirect":"https://www.pbrb.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"pclob.gov","name":"Privacy and Civil Liberties Oversight Board","redirect":"https://www.pclob.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pclob.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"presidiotunneltops.gov","name":"Presidio Trust","redirect":"https://presidiotunneltops.gov/robots.txt","sitemap":"https://wp.presidio.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://presidiotunneltops.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://wp.presidio.gov/sitemap.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"presidiotrust.gov","name":"Presidio Trust","redirect":"https://presidiotrust.gov/robots.txt","sitemap":"https://wp.presidio.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://presidiotrust.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://wp.presidio.gov/sitemap.xml"}]},{"status":500,"url":"presidiocommercial.gov","name":"Presidio Trust","redirect":"http://www.presidiocommercial.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"presidio.gov","name":"Presidio Trust","redirect":"https://presidio.gov/robots.txt","sitemap":"https://wp.presidio.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://presidio.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://wp.presidio.gov/sitemap.xml"}]},{"status":200,"url":"prc.gov","name":"Postal Regulatory Commission","redirect":"https://prc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1730174316792,"status":500,"redirect":"http://www.prc.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"pbgc.gov","name":"Pension Benefit Guaranty Corporation","redirect":"https://www.pbgc.gov/robots.txt","sitemap":"https://www.pbgc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pbgc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.pbgc.gov/sitemap.xml"}]},{"status":200,"url":"usastaffing.gov","name":"Office of Personnel Management","redirect":"https://signin.usastaffing.gov/Account/SignIn?ReturnUrl=%2Fissue%2Fwsfed%3Fwa%3Dwsignin1.0%26wtrealm%3Dhttps%253a%252f%252fusastaffing.gov%252f%26wctx%3Drm%253d0%2526id%253dpassive%2526ru%253d%25252frobots.txt%26wct%3D2025-04-05T03%253a45%253a42Z","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://signin.usastaffing.gov/Account/SignIn?ReturnUrl=%2Fissue%2Fwsfed%3Fwa%3Dwsignin1.0%26wtrealm%3Dhttps%253a%252f%252fusastaffing.gov%252f%26wctx%3Drm%253d0%2526id%253dpassive%2526ru%253d%25252frobots.txt%26wct%3D2024-06-30T02%253a17%253a04Z","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"usalearning.gov","name":"Office of Personnel Management","redirect":"https://usalearning.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.usalearning.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"usajobs.gov","name":"Office of Personnel Management","redirect":"https://www.usajobs.gov/robots.txt","sitemap":"https://www.usajobs.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usajobs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usajobs.gov/sitemap.xml"}]},{"status":200,"url":"unlocktalent.gov","name":"Office of Personnel Management","redirect":"https://www.opm.gov/data/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.opm.gov/data/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.opm.gov/data/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.unlocktalent.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"telework.gov","name":"Office of Personnel Management","redirect":"https://www.opm.gov/telework/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.opm.gov/telework/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.opm.gov/telework/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.telework.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"pmf.gov","name":"Office of Personnel Management","redirect":"https://www.pmf.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pmf.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"pac.gov","name":"Office of Personnel Management","redirect":"http://www.pac.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"opm.gov","name":"Office of Personnel Management","redirect":"https://www.opm.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.opm.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"governmentjobs.gov","name":"Office of Personnel Management","redirect":"https://www.usajobs.gov/robots.txt","sitemap":"https://www.usajobs.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usajobs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usajobs.gov/sitemap.xml"}]},{"status":500,"url":"golearn.gov","name":"Office of Personnel Management","redirect":"http://www.golearn.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"fsafeds.gov","name":"Office of Personnel Management","redirect":"https://www.fsafeds.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fegli.gov","name":"Office of Personnel Management","redirect":"https://www.opm.gov/healthcare-insurance/life-insurance","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.opm.gov/healthcare-insurance/life-insurance","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.opm.gov/healthcare-insurance/life-insurance","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://fegli.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fedshirevets.gov","name":"Office of Personnel Management","redirect":"https://www.opm.gov/fedshirevets/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.opm.gov/fedshirevets/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.opm.gov/fedshirevets/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.fedshirevets.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fedjobs.gov","name":"Office of Personnel Management","redirect":"https://www.usajobs.gov/robots.txt","sitemap":"https://www.usajobs.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usajobs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usajobs.gov/sitemap.xml"}]},{"status":200,"url":"federaljobs.gov","name":"Office of Personnel Management","redirect":"https://www.usajobs.gov/robots.txt","sitemap":"https://www.usajobs.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usajobs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usajobs.gov/sitemap.xml"}]},{"status":500,"url":"feb.gov","name":"Office of Personnel Management","redirect":"http://www.feb.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://feb.opm.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1742146243896,"status":200,"redirect":"https://feb.opm.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":404,"url":"employeeexpress.gov","name":"Office of Personnel Management","redirect":"https://www.employeeexpress.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.employeeexpress.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"e-qip.gov","name":"Office of Personnel Management","redirect":"http://www.e-qip.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"cybercareers.gov","name":"Office of Personnel Management","redirect":"https://www.opm.gov/cyber-careers/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.opm.gov/cyber-careers/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.opm.gov/cyber-careers/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.cybercareers.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"chcoc.gov","name":"Office of Personnel Management","redirect":"https://chcoc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.chcoc.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"applicationmanager.gov","name":"Office of Personnel Management","redirect":"http://www.applicationmanager.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"oge.gov","name":"Office of Government Ethics","redirect":"http://www.oge.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"integrity.gov","name":"Office of Government Ethics","redirect":"https://integrity.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.integrity.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"oshrc.gov","name":"Occupational Safety and Health Review Commission","redirect":"https://www.oshrc.gov/robots.txt","sitemap":"https://www.oshrc.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":404,"redirect":"http://oshrc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":200,"redirect":"https://www.oshrc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nrc.gov","name":"Nuclear Regulatory Commission","redirect":"https://www.nrc.gov/robots.txt","sitemap":"https://www.nrc.gov/sitemapindex.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nrc.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"nrc-gateway.gov","name":"Nuclear Regulatory Commission","redirect":"https://nrc-gateway.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1729037740707,"status":500,"redirect":"https://public-blog.nrc-gateway.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"nbrc.gov","name":"Northern Border Regional Commission","redirect":"https://www.nbrc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nbrc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"igorville.gov","name":"Non-Federal Agency","redirect":"http://www.igorville.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nitrd.gov","name":"Networking Information Technology Research and Development","redirect":"https://www.nitrd.gov/robots.txt","sitemap":"https://www.nitrd.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nitrd.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.nitrd.gov/sitemap_index.xml"}]},{"status":500,"url":"itrd.gov","name":"Networking Information Technology Research and Development","redirect":"http://www.itrd.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"ntsb.gov","name":"National Transportation Safety Board","redirect":"https://www.ntsb.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ntsb.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"nscai.gov","name":"National Security Commission on Artificial Intelligence","redirect":"http://www.nscai.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://nscai.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://cybercemetery.unt.edu/nscai/20211005220330/https://www.nscai.gov/","valid":true,"allowed":true,"sitemap":null,"source":"x-robots-tag"},{"time":1730174316792,"status":301,"redirect":"https://www.nscai.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"x-robots-tag"},{"time":1739509031548,"status":200,"redirect":"https://cybercemetery.unt.edu/nscai/20211005220330/https://www.nscai.gov/","valid":true,"allowed":true,"sitemap":null,"source":"x-robots-tag"}]},{"status":200,"url":"usap.gov","name":"National Science Foundation","redirect":"https://www.usap.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usap.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"sac.gov","name":"National Science Foundation","redirect":"https://www.sac.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.sac.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":500,"url":"research.gov","name":"National Science Foundation","redirect":"https://www.research.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":500,"redirect":"https://www.research.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":503,"redirect":"https://www.research.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"nsf.gov/staff","name":"National Science Foundation","redirect":"https://www.nsf.gov/unavailable","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.nsf.gov/staff/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://www.nsf.gov/staff/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":403,"redirect":"https://www.nsf.gov/staff","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"new.nsf.gov","name":"National Science Foundation","redirect":"https://www.nsf.gov/robots.txt","sitemap":"https://www.nsf.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://new.nsf.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://new.nsf.gov/sitemap.xml"},{"time":1737430190371,"status":200,"redirect":"https://new.nsf.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://new.nsf.gov/sitemap.xml","source":"robots.txt"},{"time":1739509031548,"status":500,"redirect":"http://www.new.nsf.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"arctic.gov","name":"National Science Foundation","redirect":"https://www.arctic.gov/robots.txt","sitemap":"https://www.arctic.gov/wp-sitemap.xml","history":[{"time":1728949876846,"status":500,"redirect":"http://www.arctic.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"nmb.gov","name":"National Mediation Board","redirect":"https://nmb.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://nmb.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"nlrb.gov","name":"National Labor Relations Board","redirect":"https://www.nlrb.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.nlrb.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"nigc.gov","name":"National Indian Gaming Commission","redirect":"https://www.nigc.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.nigc.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nga.gov","name":"National Gallery of Art","redirect":"https://www.nga.gov/robots.txt","sitemap":"https://www.nga.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nga.gov/robots.txt","valid":true,"allowed":false,"sitemap":"http://www.nga.gov/sitemap.xml"}]},{"status":200,"url":"neh.gov","name":"National Endowment for the Humanities","redirect":"https://www.neh.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://neh.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"humanities.gov","name":"National Endowment for the Humanities","redirect":"https://www.humanities.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.humanities.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1734148194754,"status":301,"redirect":"https://www.humanities.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"nea.gov","name":"National Endowment for the Arts","redirect":"https://www.arts.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.arts.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"arts.gov","name":"National Endowment for the Arts","redirect":"https://www.arts.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.arts.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"ncua.gov","name":"National Credit Union Administration","redirect":"https://ncua.gov/robots.txt","sitemap":"https://ncua.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://ncua.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://ncua.gov/sitemap.xml"}]},{"status":200,"url":"mycreditunion.gov","name":"National Credit Union Administration","redirect":"https://mycreditunion.gov/robots.txt","sitemap":"https://www.mycreditunion.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://mycreditunion.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://mycreditunion.gov/sitemap.xml"},{"time":1737430190371,"status":200,"redirect":"https://mycreditunion.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://mycreditunion.gov/sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ncd.gov","name":"National Council on Disability","redirect":"https://www.ncd.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ncd.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ncd.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.ncd.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ncd.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"ncpc.gov","name":"National Capital Planning Commission","redirect":"https://www.ncpc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ncpc.gov//robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"https://www.ncpc.govrobots.txt/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"webharvest.gov","name":"National Archives and Records Administration","redirect":"https://webharvest.gov/robots.txt","sitemap":null,"history":[{"time":1734148194754,"status":500,"redirect":"http://www.webharvest.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"wartimecontracting.gov","name":"National Archives and Records Administration","redirect":"https://cybercemetery.unt.edu/cwc/20110929213815/http:/www.wartimecontracting.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://cybercemetery.unt.edu/cwc/20110929213815/http:/www.wartimecontracting.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"trumpwhitehouse.gov","name":"National Archives and Records Administration","redirect":"https://trumpwhitehouse.archives.gov/robots.txt","sitemap":"https://trumpwhitehouse.archives.gov/sitemap/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://trumpwhitehouse.archives.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://trumpwhitehouse.archives.gov/sitemap/sitemap.xml"}]},{"status":200,"url":"trumplibrary.gov","name":"National Archives and Records Administration","redirect":"https://www.trumplibrary.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.trumplibrary.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"trumanlibrary.gov","name":"National Archives and Records Administration","redirect":"https://www.trumanlibrary.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.trumanlibrary.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"recordsmanagement.gov","name":"National Archives and Records Administration","redirect":"https://www.archives.gov/records-mgmt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.archives.gov/records-mgmt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.archives.gov/records-mgmt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://recordsmanagement.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"reaganlibrary.gov","name":"National Archives and Records Administration","redirect":"https://www.reaganlibrary.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.reaganlibrary.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"ourdocuments.gov","name":"National Archives and Records Administration","redirect":"https://www.archives.gov/milestone-documents/list","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.archives.gov/milestone-documents/list","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.archives.gov/milestone-documents/list","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://www.ourdocuments.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"obamawhitehouse.gov","name":"National Archives and Records Administration","redirect":"https://obamawhitehouse.archives.gov/robots.txt","sitemap":"https://obamawhitehouse.archives.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://obamawhitehouse.archives.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://obamawhitehouse.archives.gov/sitemap.xml"}]},{"status":200,"url":"obamalibrary.gov","name":"National Archives and Records Administration","redirect":"https://www.obamalibrary.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.obamalibrary.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"nixonlibrary.gov","name":"National Archives and Records Administration","redirect":"https://www.nixonlibrary.gov/robots.txt","sitemap":"https://www.nixonlibrary.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nixonlibrary.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.nixonlibrary.gov/sitemap.xml"}]},{"status":200,"url":"nara.gov","name":"National Archives and Records Administration","redirect":"https://www.archives.gov/robots.txt","sitemap":"https://www.archives.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.archives.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.archives.gov/sitemap.xml"}]},{"status":200,"url":"lbjlibrary.gov","name":"National Archives and Records Administration","redirect":"https://lbjlibrary.archives.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://lbjlibrary.archives.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://lbjlibrary.archives.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://www.lbjlibrary.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"jimmycarterlibrary.gov","name":"National Archives and Records Administration","redirect":"https://www.jimmycarterlibrary.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.jimmycarterlibrary.gov/robots.txt","valid":true,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.jimmycarterlibrary.gov/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"jfklibrary.gov","name":"National Archives and Records Administration","redirect":"https://jfklibrary.archives.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://jfklibrary.archives.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://jfklibrary.archives.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://jfklibrary.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"history.gov","name":"National Archives and Records Administration","redirect":"https://historyhub.history.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://historyhub.history.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"georgewbushlibrary.gov","name":"National Archives and Records Administration","redirect":"https://www.georgewbushlibrary.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.georgewbushlibrary.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"frc.gov","name":"National Archives and Records Administration","redirect":"https://www.archives.gov/frc","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.archives.gov/frc","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.archives.gov/frc","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://frc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fordlibrarymuseum.gov","name":"National Archives and Records Administration","redirect":"https://www.fordlibrarymuseum.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fordlibrarymuseum.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://fordlibrarymuseum.gov/sitemap.xml"}]},{"status":200,"url":"fcic.gov","name":"National Archives and Records Administration","redirect":"https://cybercemetery.unt.edu/fcic/20110310172443/http:/fcic.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://cybercemetery.unt.edu/fcic/20110310172443/http:/fcic.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"emergency-federal-register.gov","name":"National Archives and Records Administration","redirect":"https://www.emergency-federal-register.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://emergency-federal-register.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"eisenhowerlibrary.gov","name":"National Archives and Records Administration","redirect":"https://www.eisenhowerlibrary.gov/robots.txt","sitemap":"https://www.eisenhowerlibrary.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.eisenhowerlibrary.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.eisenhowerlibrary.gov/sitemap.xml"}]},{"status":200,"url":"dd214.gov","name":"National Archives and Records Administration","redirect":"https://www.archives.gov/veterans","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.archives.gov/veterans","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.archives.gov/veterans","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://dd214.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"clintonlibrary.gov","name":"National Archives and Records Administration","redirect":"https://www.clintonlibrary.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.clintonlibrary.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"bush41library.gov","name":"National Archives and Records Administration","redirect":"https://bush41library.archives.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://bush41library.archives.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://bush41library.archives.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://bush41library.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"archives.gov","name":"National Archives and Records Administration","redirect":"https://www.archives.gov/robots.txt","sitemap":"https://www.archives.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.archives.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.archives.gov/sitemap.xml"}]},{"status":200,"url":"911commission.gov","name":"National Archives and Records Administration","redirect":"https://911commission.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://911commission.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"9-11commission.gov","name":"National Archives and Records Administration","redirect":"https://9-11commission.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://9-11commission.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"usgeo.gov","name":"National Aeronautics and Space Administration","redirect":"http://www.usgeo.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"scijinks.gov","name":"National Aeronautics and Space Administration","redirect":"https://scijinks.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.scijinks.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nasa.gov","name":"National Aeronautics and Space Administration","redirect":"https://www.nasa.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nasa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"globe.gov","name":"National Aeronautics and Space Administration","redirect":"https://www.globe.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.globe.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":202,"redirect":"http://globe.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"earth.gov","name":"National Aeronautics and Space Administration","redirect":"https://earth.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://earth.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"udall.gov","name":"Morris K. Udall and Stewart L. Udall Foundation","redirect":"https://udall.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://udall.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ecr.gov","name":"Morris K. Udall and Stewart L. Udall Foundation","redirect":"https://www.udall.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.udall.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"mcctest.gov","name":"Millennium Challenge Corporation","redirect":"http://www.mcctest.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"mcc.gov","name":"Millennium Challenge Corporation","redirect":"https://www.mcc.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.mcc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"mspb.gov","name":"Merit Systems Protection Board","redirect":"https://www.mspb.gov/updatelinks.htm","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.mspb.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.mspb.gov/sitemapindex.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"mmc.gov","name":"Marine Mammal Commission","redirect":"https://www.mmc.gov/robots.txt","sitemap":"https://www.mmc.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.mmc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.mmc.gov/sitemap.xml"},{"time":1737430190371,"status":200,"redirect":"https://www.mmc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.mmc.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"lsc.gov","name":"Legal Services Corporation","redirect":"https://www.lsc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://lsc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"kennedy-center.gov","name":"John F. Kennedy Center for Performing Arts","redirect":"http://www.kennedy-center.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":307,"url":"jusfc.gov","name":"Japan-US Friendship Commission","redirect":"http://www.jusfc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"http://jusfc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://jusfc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"jamesmadison.gov","name":"James Madison Memorial Fellowship Foundation","redirect":"https://www.jamesmadison.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.jamesmadison.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"iaf.gov","name":"Inter-American Foundation","redirect":"http://www.iaf.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1742146243896,"status":403,"redirect":"http://www.iaf.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"pcah.gov","name":"Institute of Museum and Library Services","redirect":"http://www.pcah.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.pcah.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://pcah.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"informationliteracy.gov","name":"Institute of Museum and Library Services","redirect":"http://www.informationliteracy.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"imls.gov","name":"Institute of Museum and Library Services","redirect":"https://www.imls.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://imls.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://imls.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":500,"redirect":"http://www.imls.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"truman.gov","name":"Harry S. Truman Scholarship Foundation","redirect":"https://www.truman.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.truman.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"restorethegulf.gov","name":"Gulf Coast Ecosystem Restoration Council","redirect":"https://restorethegulf.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://restorethegulf.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"vote.gov","name":"General Services Administration","redirect":"https://vote.gov/robots.txt","sitemap":"https://vote.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://vote.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://vote.gov/sitemap.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ussm.gov","name":"General Services Administration","redirect":"https://ussm.gsa.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://ussm.gsa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://ussm.gsa.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://ussm.gsa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://ussm.gsa.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"usagov.gov","name":"General Services Administration","redirect":"https://www.usa.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usa.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usa.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://usagov.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"usability.gov","name":"General Services Administration","redirect":"https://digital.gov/topics/usability/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://digital.gov/topics/usability/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://digital.gov/topics/usability/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://usability.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"usa.gov","name":"General Services Administration","redirect":"https://www.usa.gov/robots.txt","sitemap":"https://www.usa.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usa.gov/sitemap.xml"}]},{"status":200,"url":"us.gov","name":"General Services Administration","redirect":"https://www.usa.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usa.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usa.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://us.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"thenamingcommission.gov","name":"General Services Administration","redirect":"http://www.thenamingcommission.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"statspolicy.gov","name":"General Services Administration","redirect":"https://www.statspolicy.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.statspolicy.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.statspolicy.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.statspolicy.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.statspolicy.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"sftool.gov","name":"General Services Administration","redirect":"https://sftool.gov/robots.txt","sitemap":"https://sftool.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://sftool.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://sftool.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"https://sftool.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://sftool.gov/sitemap.xml","source":"robots.txt"},{"time":1743826667324,"status":500,"redirect":"http://www.sftool.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"section508.gov","name":"General Services Administration","redirect":"https://www.section508.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.section508.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.section508.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.section508.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.section508.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"search.gov","name":"General Services Administration","redirect":"https://search.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://search.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://search.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://search.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://search.gov/sitemap.xml","source":"robots.txt"}]},{"status":404,"url":"sbst.gov","name":"General Services Administration","redirect":"https://sbst.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"x-robots-tag","history":[{"time":1726617445612,"status":404,"redirect":"https://sbst.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"sam.gov","name":"General Services Administration","redirect":"https://sam.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://sam.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"saferfederalworkforce.gov","name":"General Services Administration","redirect":"http://www.saferfederalworkforce.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://saferfederalworkforce.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"rpa.gov","name":"General Services Administration","redirect":"http://www.rpa.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"rocis.gov","name":"General Services Administration","redirect":"https://www.rocis.gov/rocis/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.rocis.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://rocis.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"reporting.gov","name":"General Services Administration","redirect":"http://www.reporting.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://reporting.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"regulations.gov","name":"General Services Administration","redirect":"https://www.regulations.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.regulations.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"reginfo.gov","name":"General Services Administration","redirect":"https://www.reginfo.gov/public/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.reginfo.gov/public/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.reginfo.gov/public/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://reginfo.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"realpropertyprofile.gov","name":"General Services Administration","redirect":"https://www.realpropertyprofile.gov/FRPPMS/robots.txt","sitemap":"https://www.realpropertyprofile.gov/FRPPMS/s/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"https://realpropertyprofile.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"https://realpropertyprofile.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1729037740707,"status":200,"redirect":"https://www.realpropertyprofile.gov/FRPPMS/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.realpropertyprofile.gov/FRPPMS/s/sitemap.xml","source":"robots.txt"},{"time":1730174316792,"status":500,"redirect":"https://www.realpropertyprofile.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"realestatesales.gov","name":"General Services Administration","redirect":"https://realestatesales.gov/login/?next=/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.realestatesales.gov","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://realestatesales.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":500,"redirect":"http://www.realestatesales.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"ptt.gov","name":"General Services Administration","redirect":"http://www.ptt.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"presidentialinnovationfellows.gov","name":"General Services Administration","redirect":"https://presidentialinnovationfellows.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://presidentialinnovationfellows.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://presidentialinnovationfellows.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://presidentialinnovationfellows.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://presidentialinnovationfellows.gov/sitemap.xml","source":"robots.txt"}]},{"status":404,"url":"ppms.gov","name":"General Services Administration","redirect":"https://ppms.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://ppms.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"plainlanguage.gov","name":"General Services Administration","redirect":"https://www.plainlanguage.gov/robots.txt","sitemap":"https://www.plainlanguage.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.plainlanguage.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://plainlanguage.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.plainlanguage.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://plainlanguage.gov/sitemap.xml","source":"robots.txt"},{"time":1730174316792,"status":200,"redirect":"https://www.plainlanguage.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"pki.gov","name":"General Services Administration","redirect":"http://www.pki.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pki-lab.gov","name":"General Services Administration","redirect":"http://www.pki-lab.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"pif.gov","name":"General Services Administration","redirect":"https://presidentialinnovationfellows.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://presidentialinnovationfellows.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://presidentialinnovationfellows.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://presidentialinnovationfellows.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://presidentialinnovationfellows.gov/sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"pic.gov","name":"General Services Administration","redirect":"http://www.pic.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":404,"redirect":"https://www.performance.gov/pic/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1734148194754,"status":500,"redirect":"https://pic.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"http://www.pic.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1742146243896,"status":500,"redirect":"https://pic.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"performance.gov","name":"General Services Administration","redirect":"https://www.performance.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.performance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.performance.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.performance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.performance.gov/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"pcscotus.gov","name":"General Services Administration","redirect":"http://www.pcscotus.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"paymentaccuracy.gov","name":"General Services Administration","redirect":"https://www.paymentaccuracy.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.paymentaccuracy.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"notify.gov","name":"General Services Administration","redirect":"https://beta.notify.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://beta.notify.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"ncmms.gov","name":"General Services Administration","redirect":"http://www.ncmms.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"madeinamerica.gov","name":"General Services Administration","redirect":"https://www.madeinamerica.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.madeinamerica.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"login.gov","name":"General Services Administration","redirect":"https://login.gov/robots.txt","sitemap":"https://login.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://login.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://login.gov/sitemap.xml"}]},{"status":200,"url":"kids.gov","name":"General Services Administration","redirect":"https://www.usa.gov/education?source=kids","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usa.gov/education?source=kids","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usa.gov/education?source=kids","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.kids.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"itdashboard.gov","name":"General Services Administration","redirect":"https://itdashboard.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":400,"redirect":"http://www.itdashboard.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"innovation.gov","name":"General Services Administration","redirect":"https://innovation.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"x-robots-tag","history":[{"time":1726617445612,"status":404,"redirect":"https://innovation.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"info.gov","name":"General Services Administration","redirect":"https://www.usa.gov/contact-us","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usa.gov/contact-us","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usa.gov/contact-us","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1729037740707,"status":302,"redirect":"https://info.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.info.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"idmanagement.gov","name":"General Services Administration","redirect":"https://www.idmanagement.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.idmanagement.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.idmanagement.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.idmanagement.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.idmanagement.gov/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"identitysandbox.gov","name":"General Services Administration","redirect":"http://www.identitysandbox.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":403,"redirect":"http://identitysandbox.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"gsaxcess.gov","name":"General Services Administration","redirect":"http://www.gsaxcess.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://gsaxcess.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://gsaxcess.govrobots.txt/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"gsatest2.gov","name":"General Services Administration","redirect":"http://www.gsatest2.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"gsaig.gov","name":"General Services Administration","redirect":"https://gsaig.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://gsaig.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"gsafleet.gov","name":"General Services Administration","redirect":"https://gsafleet.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://gsafleet.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"gsaauctions.gov","name":"General Services Administration","redirect":"https://gsaauctions.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://gsaauctions.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"gsaadvantage.gov","name":"General Services Administration","redirect":"https://gsaadvantage.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://gsaadvantage.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://gsaadvantage.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1734148194754,"status":500,"redirect":"https://gsaadvantage.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"gsa.gov","name":"General Services Administration","redirect":"https://www.gsa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.gsa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"gobiernousa.gov","name":"General Services Administration","redirect":"https://www.usa.gov/es/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.usa.gov/es/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fsrs.gov","name":"General Services Administration","redirect":"https://sam.gov/fsrs","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.fsrs.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":404,"redirect":"https://www.fsrs.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fsd.gov","name":"General Services Administration","redirect":"https://www.fsd.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fsd.gov/gsafsd_sp","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.fsd.gov/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"frpg.gov","name":"General Services Administration","redirect":"https://www.frpg.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://frpg.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"fpki.gov","name":"General Services Administration","redirect":"http://www.fpki.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"fpki-lab.gov","name":"General Services Administration","redirect":"http://www.fpki-lab.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"fpds.gov","name":"General Services Administration","redirect":"https://fpds.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.fpds.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"fpc.gov","name":"General Services Administration","redirect":"https://www.fpc.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.fpc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fpc.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.fpc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fpc.gov/sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"forms.gov","name":"General Services Administration","redirect":"https://search.usa.gov/search?affiliate=usagov_all_gov&query=government%20forms","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usa.gov/forms?utm_source=forms_gov&utm_medium=USAGov_Redirect&utm_campaign=Dec_2019_launch","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usa.gov/forms?utm_source=forms_gov&utm_medium=USAGov_Redirect&utm_campaign=Dec_2019_launch","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":302,"redirect":"https://forms.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"fmi.gov","name":"General Services Administration","redirect":"https://sftool.gov/train","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://sftool.gov/train","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://sftool.gov/train","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.fmi.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"https://sftool.gov/train","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"https://sftool.gov/train","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"http://www.fmi.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"fleet.gov","name":"General Services Administration","redirect":"http://www.fleet.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"firstgov.gov","name":"General Services Administration","redirect":"https://www.usa.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usa.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usa.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://firstgov.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"fedrooms.gov","name":"General Services Administration","redirect":"https://www.fedrooms.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://fedrooms.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fedramp.gov","name":"General Services Administration","redirect":"https://www.fedramp.gov/robots.txt","sitemap":"https://www.fedramp.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fedramp.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fedramp.gov/sitemap.xml"}]},{"status":500,"url":"fedinfo.gov","name":"General Services Administration","redirect":"http://www.fedinfo.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"fedidcard.gov","name":"General Services Administration","redirect":"https://www.gsa.gov/technology/it-contract-vehicles-and-purchasing-programs/federal-credentialing-services?gsaredirect=fedidcard","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.gsa.gov/technology/it-contract-vehicles-and-purchasing-programs/federal-credentialing-services?gsaredirect=fedidcard","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.gsa.gov/technology/it-contract-vehicles-and-purchasing-programs/federal-credentialing-services?gsaredirect=fedidcard","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://fedidcard.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fdms.gov","name":"General Services Administration","redirect":"https://www.fdms.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fdms.gov/fdms","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.fdms.gov/fdms","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"fcsm.gov","name":"General Services Administration","redirect":"https://www.fcsm.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.fcsm.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.fcsm.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fcsm.gov/sitemap.xml","source":"robots.txt"},{"time":1734148194754,"status":200,"redirect":"https://www.fcsm.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1742146243896,"status":404,"redirect":"https://www.fcsm.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"fbo.gov","name":"General Services Administration","redirect":"https://fbohome.sam.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://sam.gov/content/opportunities","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://sam.gov/content/opportunities","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://fbo.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":200,"redirect":"https://sam.gov/opportunities","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fbf.gov","name":"General Services Administration","redirect":"https://acquisitiongateway.gov/forecast","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":400,"redirect":"http://www.fbf.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.fbf.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fapiis.gov","name":"General Services Administration","redirect":"https://sam.gov/fapiis","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":400,"redirect":"http://www.fapiis.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":302,"redirect":"https://fapiis.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fai.gov","name":"General Services Administration","redirect":"https://www.fai.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fai.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"facadatabase.gov","name":"General Services Administration","redirect":"https://www.facadatabase.gov/FACA/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"https://facadatabase.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"https://facadatabase.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"faca.gov","name":"General Services Administration","redirect":"https://www.faca.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://faca.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fac.gov","name":"General Services Administration","redirect":"https://www.fac.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fac.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fac.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.fac.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fac.gov/sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"evaluation.gov","name":"General Services Administration","redirect":"https://www.evaluation.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.evaluation.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.evaluation.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.evaluation.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.evaluation.gov/sitemap.xml","source":"robots.txt"}]},{"status":404,"url":"esrs.gov","name":"General Services Administration","redirect":"https://www.esrs.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.esrs.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"ecpic.gov","name":"General Services Administration","redirect":"https://ecpic.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://ecpic.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"digitalgov.gov","name":"General Services Administration","redirect":"https://digital.gov/robots.txt","sitemap":"https://www.digital.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://digital.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://digital.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"digitaldashboard.gov","name":"General Services Administration","redirect":"http://www.digitaldashboard.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":400,"redirect":"http://www.digitaldashboard.gov","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"https://digitaldashboard.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1740965653672,"status":200,"redirect":"https://itdashboard.gov/federal-website-metrics","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"digital.gov","name":"General Services Administration","redirect":"https://digital.gov/robots.txt","sitemap":"https://www.digital.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://digital.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://digital.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"data.gov","name":"General Services Administration","redirect":"https://data.gov/robots.txt","sitemap":"https://data.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://data.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://data.gov/sitemap.xml"}]},{"status":500,"url":"csusop.gov","name":"General Services Administration","redirect":"http://www.csusop.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"cpars.gov","name":"General Services Administration","redirect":"https://cpars.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":404,"redirect":"https://cpars.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"contractdirectory.gov","name":"General Services Administration","redirect":"https://contractdirectory.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.contractdirectory.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"consumeraction.gov","name":"General Services Administration","redirect":"https://www.usa.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usa.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usa.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://consumeraction.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"computersforlearning.gov","name":"General Services Administration","redirect":"https://computersforlearning.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://computersforlearning.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"code.gov","name":"General Services Administration","redirect":"https://digital.gov/resources/requirements-for-achieving-efficiency-transparency-and-innovation-through-reusable-and-open-source-software/","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://code.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":404,"redirect":"https://code.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"x-robots-tag"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cloud.gov","name":"General Services Administration","redirect":"https://cloud.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://cloud.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://cloud.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://cloud.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://cloud.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"citizenscience.gov","name":"General Services Administration","redirect":"https://www.citizenscience.gov/robots.txt","sitemap":"https://www.citizenscience.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"https://citizenscience.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cio.gov","name":"General Services Administration","redirect":"https://www.cio.gov/robots.txt","sitemap":"https://www.cio.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cio.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.cio.gov/sitemap.xml"}]},{"status":200,"url":"challenge.gov","name":"General Services Administration","redirect":"https://www.challenge.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"https://challenge.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.challenge.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.challenge.gov/sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cfo.gov","name":"General Services Administration","redirect":"https://www.cfo.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cfo.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cfo.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.cfo.gov/sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cdo.gov","name":"General Services Administration","redirect":"https://www.cdo.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cdo.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.cdo.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.cdo.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.cdo.gov/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"cbca.gov","name":"General Services Administration","redirect":"http://www.cbca.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cao.gov","name":"General Services Administration","redirect":"http://www.cao.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"buyamerican.gov","name":"General Services Administration","redirect":"https://www.madeinamerica.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.madeinamerica.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"buyaccessible.gov","name":"General Services Administration","redirect":"https://www.section508.gov/art/","sitemap":null,"valid":true,"allowed":true,"source":"x-robots-tag","history":[{"time":1726617445612,"status":200,"redirect":"https://www.section508.gov/art/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.buyaccessible.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://buyaccessible.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"x-robots-tag"}]},{"status":200,"url":"businessusa.gov","name":"General Services Administration","redirect":"https://www.usa.gov/small-business?source=busa","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usa.gov/small-business?source=busa","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usa.gov/small-business?source=busa","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.businessusa.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"afadvantage.gov","name":"General Services Administration","redirect":"https://afadvantage.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"acquisitiongateway.gov","name":"General Services Administration","redirect":"https://acquisitiongateway.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":400,"redirect":"http://www.acquisitiongateway.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"acquisition.gov","name":"General Services Administration","redirect":"https://www.acquisition.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://login.acquisition.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"accessibility.gov","name":"General Services Administration","redirect":"https://www.section508.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.section508.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.section508.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.section508.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.section508.gov/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"400yaahc.gov","name":"General Services Administration","redirect":"https://www.400yaahc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"18f.gov","name":"General Services Administration","redirect":"https://18f.gsa.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://18f.gsa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://18f.gsa.gov/sitemap.xml"},{"time":1740965653672,"status":200,"redirect":"https://18f.gsa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"{{ site.url }}/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"sentinel.gov","name":"Federal Trade Commission","redirect":"https://www.ftc.gov/enforcement/consumer-sentinel-network","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ftc.gov/enforcement/consumer-sentinel-network","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.ftc.gov/enforcement/consumer-sentinel-network","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.sentinel.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"robodeidentidad.gov","name":"Federal Trade Commission","redirect":"https://www.robodeidentidad.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.robodeidentidad.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"protecciondelconsumidor.gov","name":"Federal Trade Commission","redirect":"http://www.protecciondelconsumidor.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"onguardonline.gov","name":"Federal Trade Commission","redirect":"https://www.consumer.ftc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.consumer.ftc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"militaryconsumer.gov","name":"Federal Trade Commission","redirect":"https://www.militaryconsumer.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.militaryconsumer.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"idtheft.gov","name":"Federal Trade Commission","redirect":"https://www.identitytheft.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.identitytheft.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.identitytheft.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.idtheft.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"identitytheft.gov","name":"Federal Trade Commission","redirect":"https://www.identitytheft.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.identitytheft.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"hsr.gov","name":"Federal Trade Commission","redirect":"https://www.ftc.gov/enforcement/premerger-notification-program","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ftc.gov/enforcement/premerger-notification-program","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.ftc.gov/enforcement/premerger-notification-program","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.hsr.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"ftc.gov","name":"Federal Trade Commission","redirect":"https://www.ftc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ftc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"econsumer.gov","name":"Federal Trade Commission","redirect":"https://econsumer.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://econsumer.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"dontserveteens.gov","name":"Federal Trade Commission","redirect":"https://consumer.ftc.gov/features/we-dont-serve-teens","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://consumer.ftc.gov/features/we-dont-serve-teens","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://consumer.ftc.gov/features/we-dont-serve-teens","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.dontserveteens.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"donotcall.gov","name":"Federal Trade Commission","redirect":"https://www.donotcall.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.donotcall.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"consumidor.gov","name":"Federal Trade Commission","redirect":"https://consumidor.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://consumidor.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"consumersentinel.gov","name":"Federal Trade Commission","redirect":"https://consumersentinel.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://consumersentinel.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"consumer.gov","name":"Federal Trade Commission","redirect":"https://consumer.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://consumer.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"alertaenlinea.gov","name":"Federal Trade Commission","redirect":"https://consumidor.ftc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://consumidor.ftc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"tsptest.gov","name":"Federal Retirement Thrift Investment Board","redirect":"http://www.tsptest.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"tsp.gov","name":"Federal Retirement Thrift Investment Board","redirect":"https://www.tsp.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"frtibtest.gov","name":"Federal Retirement Thrift Investment Board","redirect":"http://www.frtibtest.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"frtib.gov","name":"Federal Retirement Thrift Investment Board","redirect":"https://www.frtib.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"https://frtib.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.frtib.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.frtib.gov/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"exploretsp.gov","name":"Federal Retirement Thrift Investment Board","redirect":"https://www.exploretsp.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":403,"redirect":"https://www.tsp.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"uscurrency.gov","name":"Federal Reserve Board of Governors","redirect":"https://www.uscurrency.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.uscurrency.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"newmoney.gov","name":"Federal Reserve Board of Governors","redirect":"http://www.newmoney.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"frs.gov","name":"Federal Reserve Board of Governors","redirect":"http://www.frs.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"frb.gov","name":"Federal Reserve Board of Governors","redirect":"http://www.frb.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"ffiec.gov","name":"Federal Reserve Board of Governors","redirect":"https://www.ffiec.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"fedres.gov","name":"Federal Reserve Board of Governors","redirect":"http://www.fedres.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"fedpartnership.gov","name":"Federal Reserve Board of Governors","redirect":"https://www.fedpartnership.gov/Home/404?item=%2frobots&user=extranet%5cAnonymous&site=pfp","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.fedpartnership.gov/Home/404?item=%2frobots&user=extranet%5cAnonymous&site=pfp","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"federalreserveconsumerhelp.gov","name":"Federal Reserve Board of Governors","redirect":"https://www.federalreserveconsumerhelp.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.federalreserveconsumerhelp.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"federalreserve.gov","name":"Federal Reserve Board of Governors","redirect":"https://www.federalreserve.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.federalreserve.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"fbiic.gov","name":"Federal Reserve Board of Governors","redirect":"https://www.fbiic.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"billetes.gov","name":"Federal Reserve Board of Governors","redirect":"https://www.uscurrency.gov/es","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.uscurrency.gov/es","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.uscurrency.gov/es","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.billetes.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"permitting.gov","name":"Federal Permitting Improvement Steering Council","redirect":"https://www.permitting.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.permits.performance.gov/error404","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://www.permits.performance.gov/error404","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"fpisc.gov","name":"Federal Permitting Improvement Steering Council","redirect":"https://www.permitting.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.permits.performance.gov/error404","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":404,"redirect":"https://www.permits.performance.gov/error404","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fmshrc.gov","name":"Federal Mine Safety and Health Review Commission","redirect":"https://fmshrc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.fmshrc.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"fmcs.gov","name":"Federal Mediation and Conciliation Service","redirect":"https://www.fmcs.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.fmcs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fmcs.gov/sitemap_index.xml"}]},{"status":500,"url":"adr.gov","name":"Federal Mediation and Conciliation Service","redirect":"http://www.adr.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://adr.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://adr.gov/wp-sitemap.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"fmc.gov","name":"Federal Maritime Commission","redirect":"https://www.fmc.gov/robots.txt","sitemap":"https://www.fmc.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fmc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fmc.gov/sitemap_index.xml"}]},{"status":200,"url":"flra.gov","name":"Federal Labor Relations Authority","redirect":"http://www.flra.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.flra.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"mortgagetranslations.gov","name":"Federal Housing Finance Agency","redirect":"https://www.fhfa.gov/mortgage-translations","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.fhfa.gov/mortgage-translations","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.mortgagetranslations.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"fhfaoig.gov","name":"Federal Housing Finance Agency","redirect":"https://www.fhfaoig.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":404,"redirect":"http://www.fhfaoig.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"fhfa.gov","name":"Federal Housing Finance Agency","redirect":"https://www.fhfa.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ferc.gov","name":"Federal Energy Regulatory Commission","redirect":"https://ferc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://ferc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"fec.gov","name":"Federal Election Commission","redirect":"https://www.fec.gov/robots.txt","sitemap":"https://www.fec.gov/sitemap-wagtail.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fec.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fec.gov/sitemap-wagtail.xml"}]},{"status":200,"url":"ofia.gov","name":"Federal Deposit Insurance Corporation","redirect":"https://www.ofia.gov/robots.txt","sitemap":"https://www.ofia.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"myfdic.gov","name":"Federal Deposit Insurance Corporation","redirect":"http://www.myfdic.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.myfdic.gov","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://myfdic.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"fdicseguro.gov","name":"Federal Deposit Insurance Corporation","redirect":"http://www.fdicseguro.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.fdicseguro.gov","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://www.fdic.gov/espanol/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fdicoig.gov","name":"Federal Deposit Insurance Corporation","redirect":"https://www.fdicoig.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fdicoig.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"fdicig.gov","name":"Federal Deposit Insurance Corporation","redirect":"http://www.fdicig.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"fdicconnect.gov","name":"Federal Deposit Insurance Corporation","redirect":"http://www.fdicconnect.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"fdic.gov","name":"Federal Deposit Insurance Corporation","redirect":"https://fdic.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"economicinclusion.gov","name":"Federal Deposit Insurance Corporation","redirect":"http://www.economicinclusion.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.economicinclusion.gov","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":404,"redirect":"https://www.fdic.gov/analysis/household-survey//robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"https://www.fdic.gov/household-survey/household-survey-archives","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"getinternet.gov","name":"Federal Communications Commission","redirect":"https://getinternet.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":403,"redirect":"http://www.getinternet.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fcc.gov","name":"Federal Communications Commission","redirect":"https://www.fcc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":403,"redirect":"http://www.fcc.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"broadbandmap.gov","name":"Federal Communications Commission","redirect":"https://broadbandmap.fcc.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://broadbandmap.fcc.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://broadbandmap.fcc.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.broadbandmap.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"affordableconnectivity.gov","name":"Federal Communications Commission","redirect":"http://www.affordableconnectivity.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://affordableconnectivity.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://affordableconnectivity.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"acpbenefit.gov","name":"Federal Communications Commission","redirect":"https://www.affordableconnectivity.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.affordableconnectivity.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://www.affordableconnectivity.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"accedeainternet.gov","name":"Federal Communications Commission","redirect":"https://www.getinternet.gov/apply?ln=U3BhbmlzaA%3D%3D&awsl=accedeainternet","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":403,"redirect":"http://www.accedeainternet.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.accedeainternet.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"fcsic.gov","name":"Farm Credit Administration","redirect":"https://www.fcsic.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.fcsic.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fca.gov","name":"Farm Credit Administration","redirect":"https://www.fca.gov/robots.txt","sitemap":"https://www.fca.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fca.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fca.gov/sitemap.xml"}]},{"status":200,"url":"exim.gov","name":"Export/Import Bank of the U.S.","redirect":"https://exim.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://exim.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"whitehousedrugpolicy.gov","name":"Executive Office of the President","redirect":"http://www.whitehousedrugpolicy.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/ondcp/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/ondcp/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://whitehousedrugpolicy.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"https://www.whitehouse.gov/ondcp/","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"whitehouse.gov","name":"Executive Office of the President","redirect":"https://www.whitehouse.gov/robots.txt","sitemap":"https://www.whitehouse.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.whitehouse.gov/sitemap_index.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"wh.gov","name":"Executive Office of the President","redirect":"https://www.whitehouse.gov/robots.txt","sitemap":"https://www.whitehouse.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.whitehouse.gov/sitemap_index.xml"}]},{"status":200,"url":"ustr.gov","name":"Executive Office of the President","redirect":"https://ustr.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://ustr.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"usds.gov","name":"Executive Office of the President","redirect":"https://www.usds.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usds.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://usds.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.usds.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://usds.gov/sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"usdigitalservice.gov","name":"Executive Office of the President","redirect":"https://www.usds.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usds.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://usdigitalservice.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.usds.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://usdigitalservice.gov/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"unitedwestand.gov","name":"Executive Office of the President","redirect":"http://www.unitedwestand.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://unitedwestand.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":404,"redirect":"https://unitedwestand.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"https://www.whitehouse.gov/","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"reprorights.gov","name":"Executive Office of the President","redirect":"http://www.reprorights.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://reproductiverights.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://reproductiverights.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.reprorights.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":200,"redirect":"https://reproductiverights.gov/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"reproductiverights.gov","name":"Executive Office of the President","redirect":"http://www.reproductiverights.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://reproductiverights.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://reproductiverights.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":200,"redirect":"https://reproductiverights.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"reproductiveaccess.gov","name":"Executive Office of the President","redirect":"http://www.reproductiveaccess.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://reproductiverights.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://reproductiverights.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://reproductiveaccess.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":200,"redirect":"https://reproductiverights.gov/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"reproaccess.gov","name":"Executive Office of the President","redirect":"http://www.reproaccess.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://reproductiverights.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://reproductiverights.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.reproaccess.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":200,"redirect":"https://reproductiverights.gov/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"quantum.gov","name":"Executive Office of the President","redirect":"https://www.quantum.gov/robots.txt","sitemap":"https://www.quantum.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.quantum.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.quantum.gov/sitemap_index.xml"}]},{"status":500,"url":"publicserviceloanforgiveness.gov","name":"Executive Office of the President","redirect":"http://www.publicserviceloanforgiveness.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/publicserviceloanforgiveness/?utm_source=publicserviceloanforgiveness.gov","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/publicserviceloanforgiveness/?utm_source=www.publicserviceloanforgiveness.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.publicserviceloanforgiveness.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"https://www.whitehouse.gov/publicserviceloanforgiveness/?utm_source=www.publicserviceloanforgiveness.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"pslf.gov","name":"Executive Office of the President","redirect":"http://www.pslf.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/publicserviceloanforgiveness/?utm_source=pslf.gov","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/publicserviceloanforgiveness/?utm_source=www.pslf.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.pslf.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"https://www.whitehouse.gov/publicserviceloanforgiveness/?utm_source=www.pslf.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"pitctestdomain.gov","name":"Executive Office of the President","redirect":"http://www.pitctestdomain.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pitc.gov","name":"Executive Office of the President","redirect":"http://www.pitc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"pci.gov","name":"Executive Office of the President","redirect":"http://www.pci.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"ostp.gov","name":"Executive Office of the President","redirect":"https://www.whitehouse.gov/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/ostp/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/ostp/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://ostp.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"ondcp.gov","name":"Executive Office of the President","redirect":"https://www.whitehouse.gov/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/ondcp/?utm_source=ondcp.gov","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/ondcp/?utm_source=www.ondcp.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.ondcp.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"omb.gov","name":"Executive Office of the President","redirect":"https://www.whitehouse.gov/omb/?utm_source=www.omb.gov","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/omb/?utm_source=omb.gov","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/omb/?utm_source=www.omb.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.omb.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"nepa.gov","name":"Executive Office of the President","redirect":"https://nepa.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"nano.gov","name":"Executive Office of the President","redirect":"https://www.nano.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nano.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"max.gov","name":"Executive Office of the President","redirect":"https://max.omb.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://max.omb.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"investinginamerica.gov","name":"Executive Office of the President","redirect":"http://www.investinginamerica.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/invest/?utm_source=investinginamerica.gov","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/invest/?utm_source=www.investinginamerica.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.investinginamerica.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"https://www.whitehouse.gov/invest/?utm_source=www.investinginamerica.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"investinamerica.gov","name":"Executive Office of the President","redirect":"http://www.investinamerica.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/invest/?utm_source=investinamerica.gov","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/invest/?utm_source=www.investinamerica.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.investinamerica.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"https://www.whitehouse.gov/invest/?utm_source=www.investinamerica.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"invest.gov","name":"Executive Office of the President","redirect":"http://www.invest.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/invest/?utm_source=invest.gov","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/invest/?utm_source=www.invest.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.invest.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"https://www.whitehouse.gov/invest/?utm_source=www.invest.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"invertir.gov","name":"Executive Office of the President","redirect":"http://www.invertir.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/es/invertir/?utm_source=invertir.gov","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/es/invertir/?utm_source=www.invertir.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.invertir.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"https://www.whitehouse.gov/es/invertir/?utm_source=www.invertir.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"eop.gov","name":"Executive Office of the President","redirect":"https://www.whitehouse.gov/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://eop.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"environmentaljustice.gov","name":"Executive Office of the President","redirect":"http://www.environmentaljustice.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":403,"redirect":"https://environmentaljustice.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://environmentaljustice.gov/robots.txt","valid":true,"allowed":true,"sitemap":"http://environmentaljustice.gov/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"ej.gov","name":"Executive Office of the President","redirect":"http://www.ej.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.environmentaljustice.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.environmentaljustice.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://ej.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"https://www.environmentaljustice.gov/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"earmarks.gov","name":"Executive Office of the President","redirect":"https://www.whitehouse.gov/omb/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/omb/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/omb/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":302,"redirect":"https://earmarks.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"crisisnextdoor.gov","name":"Executive Office of the President","redirect":"http://www.crisisnextdoor.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cdc.gov/overdose-prevention/index.html","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cdc.gov/overdose-prevention/index.html","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":302,"redirect":"https://crisisnextdoor.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"https://www.cdc.gov/overdose-prevention/index.html","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"cleanenergy.gov","name":"Executive Office of the President","redirect":"http://www.cleanenergy.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.whitehouse.gov/cleanenergy/robots.txt?utm_source=cleanenergy.gov","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://www.whitehouse.gov/cleanenergy/robots.txt?utm_source=www.cleanenergy.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"childtaxcredit.gov","name":"Executive Office of the President","redirect":"http://www.childtaxcredit.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/child-tax-credit/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/child-tax-credit/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://childtaxcredit.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"https://www.whitehouse.gov/child-tax-credit/","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"buildbackbetter.gov","name":"Executive Office of the President","redirect":"http://www.buildbackbetter.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://buildbackbetter.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"https://www.whitehouse.gov/","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"build.gov","name":"Executive Office of the President","redirect":"http://www.build.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.whitehouse.gov/build/robots.txt?utm_source=build.gov","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://www.whitehouse.gov/build/robots.txt?utm_source=www.build.gov","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"budget.gov","name":"Executive Office of the President","redirect":"http://www.budget.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.whitehouse.gov/omb/budget/?utm_source=redirect&utm_medium=social","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whitehouse.gov/omb/budget/?utm_source=redirect&utm_medium=social","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://budget.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":200,"redirect":"https://www.whitehouse.gov/omb/budget/?utm_source=redirect&utm_medium=social","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"ai.gov","name":"Executive Office of the President","redirect":"https://www.whitehouse.gov/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://ai.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://ai.gov/sitemap.xml"},{"time":1739509031548,"status":200,"redirect":"https://ai.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://ai.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"eeoc.gov","name":"Equal Employment Opportunity Commission","redirect":"https://www.eeoc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.eeoc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"urbanwaters.gov","name":"Environmental Protection Agency","redirect":"https://www.epa.gov/urbanwaterspartners","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.epa.gov/urbanwaterspartners","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.epa.gov/urbanwaterspartners","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://urbanwaters.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"sustainability.gov","name":"Environmental Protection Agency","redirect":"https://www.sustainability.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.sustainability.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"relocatefeds.gov","name":"Environmental Protection Agency","redirect":"https://www.epa.gov/relocatefeds","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.epa.gov/relocatefeds","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.epa.gov/relocatefeds","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://relocatefeds.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"greengov.gov","name":"Environmental Protection Agency","redirect":"http://www.greengov.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":503,"redirect":"https://greengov.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"glnpo.gov","name":"Environmental Protection Agency","redirect":"http://www.glnpo.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"frtr.gov","name":"Environmental Protection Agency","redirect":"https://frtr.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"http://frtr.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.frtr.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"foiaonline.gov","name":"Environmental Protection Agency","redirect":"http://www.foiaonline.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"fedcenter.gov","name":"Environmental Protection Agency","redirect":"http://www.fedcenter.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"http://fedcenter.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"epaoig.gov","name":"Environmental Protection Agency","redirect":"https://www.epaoig.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.epaoig.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"epa.gov","name":"Environmental Protection Agency","redirect":"https://www.epa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.epa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"energystar.gov","name":"Environmental Protection Agency","redirect":"https://www.energystar.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.energystar.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"cbi-epa.gov","name":"Environmental Protection Agency","redirect":"http://www.cbi-epa.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"airnow.gov","name":"Environmental Protection Agency","redirect":"https://www.airnow.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://airnow.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://airnow.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1740965653672,"status":200,"redirect":"https://www.airnow.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"airknowledge.gov","name":"Environmental Protection Agency","redirect":"http://www.airknowledge.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"votebymail.gov","name":"Election Assistance Commission","redirect":"https://www.votebymail.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://votebymail.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"helpamericavote.gov","name":"Election Assistance Commission","redirect":"https://www.helpamericavote.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://helpamericavote.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"eac.gov","name":"Election Assistance Commission","redirect":"https://www.eac.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://eac.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"ugov.gov","name":"Director of National Intelligence","redirect":"http://www.ugov.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"osis.gov","name":"Director of National Intelligence","redirect":"http://www.osis.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"odni.gov","name":"Director of National Intelligence","redirect":"https://www.odni.gov/robots.txt","sitemap":"https://www.dni.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.odni.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.odni.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"nctc.gov","name":"Director of National Intelligence","redirect":"https://www.dni.gov/index.php/nctc-home","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dni.gov/index.php/nctc-home","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dni.gov/index.php/nctc-home","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.nctc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"ncsc.gov","name":"Director of National Intelligence","redirect":"https://www.dni.gov/index.php/ncsc-home","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dni.gov/index.php/ncsc-home","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dni.gov/index.php/ncsc-home","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.ncsc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"ncix.gov","name":"Director of National Intelligence","redirect":"http://www.ncix.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"ise.gov","name":"Director of National Intelligence","redirect":"https://www.dni.gov/index.php/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dni.gov/index.php/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dni.gov/index.php/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.ise.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"intelligence.gov","name":"Director of National Intelligence","redirect":"https://www.intelligence.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.intelligence.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"intelink.gov","name":"Director of National Intelligence","redirect":"http://www.intelink.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"intel.gov","name":"Director of National Intelligence","redirect":"https://www.intel.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.intel.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"icjointduty.gov","name":"Director of National Intelligence","redirect":"https://www.dni.gov/index.php/careers/joint-duty","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dni.gov/index.php/careers/joint-duty","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dni.gov/index.php/careers/joint-duty","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.icjointduty.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"iarpa.gov","name":"Director of National Intelligence","redirect":"https://www.iarpa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.iarpa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"iarpa-ideas.gov","name":"Director of National Intelligence","redirect":"https://iarpa-ideas.gov/client/signin.aspx","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://iarpa-ideas.gov/client/signin.aspx","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://iarpa-ideas.gov/client/signin.aspx","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://iarpa-ideas.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"famep.gov","name":"Director of National Intelligence","redirect":"http://www.famep.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"dni.gov","name":"Director of National Intelligence","redirect":"https://www.dni.gov/robots.txt","sitemap":"https://www.dni.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.dni.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.dni.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"ce-ncsc.gov","name":"Director of National Intelligence","redirect":"http://www.ce-ncsc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"workplace.gov","name":"Department of the Treasury","redirect":"http://www.workplace.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"wizard.gov","name":"Department of the Treasury","redirect":"https://wizard.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://wizard.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ustreas.gov","name":"Department of the Treasury","redirect":"https://home.treasury.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://home.treasury.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://home.treasury.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.treasury.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":403,"url":"usmint.gov","name":"Department of the Treasury","redirect":"https://www.usmint.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usmint.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usmint.gov/sitemap_index.xml"},{"time":1734148194754,"status":200,"redirect":"https://www.usmint.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usmint.gov/sitemap_index.xml","source":"robots.txt"}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"usdebitcard.gov","name":"Department of the Treasury","redirect":"https://www.usdebitcard.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://usdebitcard.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"usaspending.gov","name":"Department of the Treasury","redirect":"https://www.usaspending.gov/robots.txt","sitemap":"https://www.usaspending.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usaspending.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usaspending.gov/sitemap.xml"}]},{"status":500,"url":"twai.gov","name":"Department of the Treasury","redirect":"http://www.twai.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ttlplus.gov","name":"Department of the Treasury","redirect":"http://www.ttlplus.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ttbonline.gov","name":"Department of the Treasury","redirect":"https://ttbonline.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.ttbonline.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ttb.gov","name":"Department of the Treasury","redirect":"https://www.ttb.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":400,"redirect":"http://ttb.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"treasuryscams.gov","name":"Department of the Treasury","redirect":"https://treasuryscams.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://treasuryscams.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"treasuryhunt.gov","name":"Department of the Treasury","redirect":"https://www.treasuryhunt.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://treasuryhunt.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"treasuryecm.gov","name":"Department of the Treasury","redirect":"http://www.treasuryecm.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"treasurydirect.gov","name":"Department of the Treasury","redirect":"https://treasurydirect.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://treasurydirect.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"treasuryauctions.gov","name":"Department of the Treasury","redirect":"https://treasuryauctions.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://treasuryauctions.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"treasuryauction.gov","name":"Department of the Treasury","redirect":"http://www.treasuryauction.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"treasury.gov","name":"Department of the Treasury","redirect":"https://home.treasury.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://home.treasury.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://home.treasury.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.treasury.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"treaslockbox.gov","name":"Department of the Treasury","redirect":"https://treaslockbox.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://treaslockbox.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"treas.gov","name":"Department of the Treasury","redirect":"https://home.treasury.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://home.treasury.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://home.treasury.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.treasury.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"transparency.gov","name":"Department of the Treasury","redirect":"http://www.transparency.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"tigtanet.gov","name":"Department of the Treasury","redirect":"http://www.tigtanet.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"tigta.gov","name":"Department of the Treasury","redirect":"https://www.tigta.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.tigta.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"tcis.gov","name":"Department of the Treasury","redirect":"http://www.tcis.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"tax.gov","name":"Department of the Treasury","redirect":"http://www.tax.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"taaps.gov","name":"Department of the Treasury","redirect":"https://taaps.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://taaps.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"slgs.gov","name":"Department of the Treasury","redirect":"https://slgs.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://slgs.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"sigtarp.gov","name":"Department of the Treasury","redirect":"http://www.sigtarp.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.sigtarp.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"sigpr.gov","name":"Department of the Treasury","redirect":"https://home.treasury.gov/services/report-fraud-waste-and-abuse","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.sigpr.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://www.sigpr.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":404,"url":"savingsbondwizard.gov","name":"Department of the Treasury","redirect":"https://www.savingsbondwizard.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://savingsbondwizard.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"savingsbonds.gov","name":"Department of the Treasury","redirect":"https://savingsbonds.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://savingsbonds.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://savingsbonds.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"savingsbond.gov","name":"Department of the Treasury","redirect":"https://savingsbond.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://savingsbond.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"qatesttwai.gov","name":"Department of the Treasury","redirect":"http://www.qatesttwai.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"pay.gov","name":"Department of the Treasury","redirect":"https://www.pay.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pay.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"patriotbonds.gov","name":"Department of the Treasury","redirect":"http://www.patriotbonds.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ots.gov","name":"Department of the Treasury","redirect":"https://www.ots.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"occnet.gov","name":"Department of the Treasury","redirect":"http://www.occnet.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"occhelps.gov","name":"Department of the Treasury","redirect":"https://occhelps.gov/robots.txt","sitemap":"https://helpwithmybank.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://occhelps.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://helpwithmybank.gov/sitemap.xml"}]},{"status":500,"url":"occ.gov","name":"Department of the Treasury","redirect":"http://www.occ.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"navycash.gov","name":"Department of the Treasury","redirect":"https://www.navycash.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://fiscal.treasury.gov/navy-cash/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://fiscal.treasury.gov/navy-cash/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://navycash.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":200,"redirect":"https://fiscal.treasury.gov/navy-cash/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"nationalbanknet.gov","name":"Department of the Treasury","redirect":"https://www.nationalbanknet.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1730174316792,"status":404,"redirect":"https://www.nationalbanknet.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":500,"redirect":"https://www.banknet.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"nationalbankhelp.gov","name":"Department of the Treasury","redirect":"https://nationalbankhelp.gov/robots.txt","sitemap":"https://helpwithmybank.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://nationalbankhelp.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://helpwithmybank.gov/sitemap.xml"}]},{"status":404,"url":"nationalbank.gov","name":"Department of the Treasury","redirect":"https://www.nationalbank.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"mytreasury.gov","name":"Department of the Treasury","redirect":"http://www.mytreasury.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"myra.gov","name":"Department of the Treasury","redirect":"http://www.myra.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"mymoney.gov","name":"Department of the Treasury","redirect":"https://www.mymoney.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.mymoney.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"myira.gov","name":"Department of the Treasury","redirect":"http://www.myira.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"msb.gov","name":"Department of the Treasury","redirect":"http://www.msb.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"moneyfactorystore.gov","name":"Department of the Treasury","redirect":"https://www.usmint.gov/bureau-of-engraving.html","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":403,"redirect":"https://catalog.usmint.gov/bureau-of-engraving.html","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.moneyfactorystore.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"moneyfactory.gov","name":"Department of the Treasury","redirect":"https://www.bep.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.bep.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://www.moneyfactory.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"mha.gov","name":"Department of the Treasury","redirect":"https://home.treasury.gov/data/troubled-assets-relief-program/housing","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://home.treasury.gov/data/troubled-assets-relief-program/housing","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://home.treasury.gov/data/troubled-assets-relief-program/housing","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.mha.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"makinghomeaffordable.gov","name":"Department of the Treasury","redirect":"https://home.treasury.gov/data/troubled-assets-relief-program/housing","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://home.treasury.gov/data/troubled-assets-relief-program/housing","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://home.treasury.gov/data/troubled-assets-relief-program/housing","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.makinghomeaffordable.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"its.gov","name":"Department of the Treasury","redirect":"https://its.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://its.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"irsvideos.gov","name":"Department of the Treasury","redirect":"http://www.irsvideos.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"irssales.gov","name":"Department of the Treasury","redirect":"https://www.irsauctions.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.irsauctions.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.irssales.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"irsnet.gov","name":"Department of the Treasury","redirect":"http://www.irsnet.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"irsauctions.gov","name":"Department of the Treasury","redirect":"https://www.irsauctions.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"irs.gov","name":"Department of the Treasury","redirect":"https://www.irs.gov/robots.txt","sitemap":"https://www.irs.gov/sitemap.xml","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ipp.gov","name":"Department of the Treasury","redirect":"https://www.ipp.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ipp.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"ipac.gov","name":"Department of the Treasury","redirect":"http://www.ipac.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"helpwithmymortgagebank.gov","name":"Department of the Treasury","redirect":"https://helpwithmymortgagebank.gov/robots.txt","sitemap":"https://helpwithmybank.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://helpwithmymortgagebank.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://helpwithmybank.gov/sitemap.xml"}]},{"status":200,"url":"helpwithmymortgage.gov","name":"Department of the Treasury","redirect":"https://helpwithmymortgage.gov/robots.txt","sitemap":"https://helpwithmybank.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://helpwithmymortgage.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://helpwithmybank.gov/sitemap.xml"}]},{"status":200,"url":"helpwithmycreditcardbank.gov","name":"Department of the Treasury","redirect":"https://helpwithmycreditcardbank.gov/robots.txt","sitemap":"https://helpwithmybank.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://helpwithmycreditcardbank.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://helpwithmybank.gov/sitemap.xml"}]},{"status":200,"url":"helpwithmycreditcard.gov","name":"Department of the Treasury","redirect":"https://helpwithmycreditcard.gov/robots.txt","sitemap":"https://helpwithmybank.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://helpwithmycreditcard.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://helpwithmybank.gov/sitemap.xml"}]},{"status":200,"url":"helpwithmycheckingaccount.gov","name":"Department of the Treasury","redirect":"https://helpwithmycheckingaccount.gov/robots.txt","sitemap":"https://helpwithmybank.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://helpwithmycheckingaccount.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://helpwithmybank.gov/sitemap.xml"}]},{"status":500,"url":"helpwithmybank.gov","name":"Department of the Treasury","redirect":"http://www.helpwithmybank.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"godirect.gov","name":"Department of the Treasury","redirect":"https://www.godirect.gov/gpw/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.godirect.gov/maintenance/redirect.html","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fsoc.gov","name":"Department of the Treasury","redirect":"https://home.treasury.gov/policy-issues/financial-markets-financial-institutions-and-fiscal-service/fsoc","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://home.treasury.gov/policy-issues/financial-markets-financial-institutions-and-fiscal-service/fsoc","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://home.treasury.gov/policy-issues/financial-markets-financial-institutions-and-fiscal-service/fsoc","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.fsoc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"fincen.gov","name":"Department of the Treasury","redirect":"http://www.fincen.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"financialstability.gov","name":"Department of the Treasury","redirect":"https://home.treasury.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://home.treasury.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"financialresearch.gov","name":"Department of the Treasury","redirect":"https://www.financialresearch.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.financialresearch.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"ffb.gov","name":"Department of the Treasury","redirect":"https://www.ffb.gov/robots.txt","sitemap":"{{ site.url }}/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ffb.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"fedinvest.gov","name":"Department of the Treasury","redirect":"https://www.fedinvest.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://treasurydirect.gov/government/federal-investments-program/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://treasurydirect.gov/government/federal-investments-program/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://fedinvest.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":200,"redirect":"https://treasurydirect.gov/government/federal-investments-program/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"federalspending.gov","name":"Department of the Treasury","redirect":"http://www.federalspending.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"federalinvestments.gov","name":"Department of the Treasury","redirect":"https://www.treasurydirect.gov/government/federal-investments-program/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://federalinvestments.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://federalinvestments.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"eyenote.gov","name":"Department of the Treasury","redirect":"https://www.bep.gov/services/currency-accessibility/mobile-applications","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.bep.gov/services/currency-accessibility/mobile-applications","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://www.eyenote.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"ethicsburg.gov","name":"Department of the Treasury","redirect":"http://www.ethicsburg.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":400,"url":"eta-find.gov","name":"Department of the Treasury","redirect":"http://www.eta-find.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"eftps.gov","name":"Department of the Treasury","redirect":"https://www.eftps.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.eftps.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"eaglecash.gov","name":"Department of the Treasury","redirect":"https://www.eaglecash.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://fiscal.treasury.gov/eaglecash/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://fiscal.treasury.gov/eaglecash/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://eaglecash.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":200,"redirect":"https://fiscal.treasury.gov/eaglecash/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"directoasucuenta.gov","name":"Department of the Treasury","redirect":"https://www.godirect.gov/gpw/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://godirect.gov/maintenance/redirect.html","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"comptrollerofthecurrency.gov","name":"Department of the Treasury","redirect":"https://comptrollerofthecurrency.gov/robots.txt","sitemap":"https://www.occ.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://comptrollerofthecurrency.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.occ.gov/sitemap.xml"}]},{"status":500,"url":"complaintreferralexpress.gov","name":"Department of the Treasury","redirect":"http://www.complaintreferralexpress.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"cdfifund.gov","name":"Department of the Treasury","redirect":"https://www.cdfifund.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cdfifund.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"ccac.gov","name":"Department of the Treasury","redirect":"https://www.ccac.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ccac.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"bondpro.gov","name":"Department of the Treasury","redirect":"https://www.bondpro.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://bondpro.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"bfem.gov","name":"Department of the Treasury","redirect":"http://www.bfem.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"bep.gov","name":"Department of the Treasury","redirect":"https://www.bep.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"banknet.gov","name":"Department of the Treasury","redirect":"http://www.banknet.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"bankhelp.gov","name":"Department of the Treasury","redirect":"https://bankhelp.gov/robots.txt","sitemap":"https://helpwithmybank.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://bankhelp.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://helpwithmybank.gov/sitemap.xml"}]},{"status":200,"url":"bankcustomerassistance.gov","name":"Department of the Treasury","redirect":"https://bankcustomerassistance.gov/robots.txt","sitemap":"https://helpwithmybank.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://bankcustomerassistance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://helpwithmybank.gov/sitemap.xml"}]},{"status":200,"url":"bankcustomer.gov","name":"Department of the Treasury","redirect":"https://bankcustomer.gov/robots.txt","sitemap":"https://helpwithmybank.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://bankcustomer.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://helpwithmybank.gov/sitemap.xml"}]},{"status":200,"url":"bankanswers.gov","name":"Department of the Treasury","redirect":"https://bankanswers.gov/robots.txt","sitemap":"https://helpwithmybank.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://bankanswers.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://helpwithmybank.gov/sitemap.xml"}]},{"status":200,"url":"ayudaconmibanco.gov","name":"Department of the Treasury","redirect":"https://ayudaconmibanco.gov/robots.txt","sitemap":"https://helpwithmybank.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://ayudaconmibanco.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://helpwithmybank.gov/sitemap.xml"}]},{"status":404,"url":"asap.gov","name":"Department of the Treasury","redirect":"https://asap.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://asap.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"americathebeautifulquarters.gov","name":"Department of the Treasury","redirect":"https://www.americathebeautifulquarters.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usmint.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usmint.gov/sitemap_index.xml"},{"time":1734148194754,"status":200,"redirect":"https://www.usmint.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.usmint.gov/sitemap_index.xml","source":"robots.txt"}]},{"status":404,"url":"ama.gov","name":"Department of the Treasury","redirect":"https://www.ama.gov/amagov","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"https://www.ama.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"wlci.gov","name":"Department of the Interior","redirect":"https://wlci.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://wlci.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":404,"redirect":"https://wlci.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":500,"redirect":"http://www.wlci.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"watermonitor.gov","name":"Department of the Interior","redirect":"https://watermonitor.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://watermonitor.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":404,"redirect":"https://watermonitor.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":500,"redirect":"http://www.watermonitor.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://watermonitor.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"https://watermonitor.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"volunteer.gov","name":"Department of the Interior","redirect":"https://www.volunteer.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":522,"redirect":"http://volunteer.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"volcano.gov","name":"Department of the Interior","redirect":"https://www.usgs.gov/natural-hazards/volcano-hazards/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usgs.gov/natural-hazards/volcano-hazards/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usgs.gov/natural-hazards/volcano-hazards/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://volcano.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":200,"redirect":"https://www.usgs.gov/natural-hazards/volcano-hazards/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"https://www.volcano.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"utahfireinfo.gov","name":"Department of the Interior","redirect":"https://utah-fire-info-utahdnr.hub.arcgis.com/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":"https://utahfireinfo.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://utah-fire-info-utahdnr.hub.arcgis.com/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.utahfireinfo.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"usgs.gov","name":"Department of the Interior","redirect":"https://www.usgs.gov/robots.txt","sitemap":"https://www.usgs.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usgs.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1736981098582,"status":200,"redirect":"https://www.usgs.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1737430190371,"status":500,"redirect":"https://www.usgs.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"usbr.gov","name":"Department of the Interior","redirect":"http://www.usbr.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"sierrawild.gov","name":"Department of the Interior","redirect":"https://www.sierrawild.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.sierrawild.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"sciencebase.gov","name":"Department of the Interior","redirect":"https://www.sciencebase.gov/robots.txt","sitemap":null,"valid":true,"allowed":false,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.sciencebase.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.sciencebase.gov/sitemaps/sitemap_index.xml"},{"time":1729037740707,"status":200,"redirect":"https://www.sciencebase.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.sciencebase.gov/sitemaps/sitemap_index.xml","source":"robots.txt"},{"time":1730174316792,"status":500,"redirect":"https://www.sciencebase.gov/catalog/","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"https://www.sciencebase.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.sciencebase.gov/sitemaps/sitemap_index.xml","source":"robots.txt"},{"time":1739509031548,"status":503,"redirect":"https://www.sciencebase.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":200,"redirect":"https://www.sciencebase.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.sciencebase.gov/sitemaps/sitemap_index.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"salmonrecovery.gov","name":"Department of the Interior","redirect":"https://salmonrecovery.gov/robots.txt","sitemap":"https://www.salmonrecovery.gov/sitemap.xml","history":[{"time":1726617445612,"status":404,"redirect":"http://www.salmonrecovery.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.salmonrecovery.gov","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":404,"redirect":"https://salmonrecovery.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"safecom.gov","name":"Department of the Interior","redirect":"https://safecom.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://safecom.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"rivers.gov","name":"Department of the Interior","redirect":"https://rivers.gov/robots.txt","sitemap":"https://www.fws.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1734148194754,"status":500,"redirect":"http://www.rivers.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://rivers.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"reportband.gov","name":"Department of the Interior","redirect":"http://www.reportband.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.pwrc.usgs.gov/BBL/bblretrv/index.cfm","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.reportband.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"piedrasblancas.gov","name":"Department of the Interior","redirect":"http://www.piedrasblancas.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"osmre.gov","name":"Department of the Interior","redirect":"http://www.osmre.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"osm.gov","name":"Department of the Interior","redirect":"http://www.osm.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"onrr.gov","name":"Department of the Interior","redirect":"https://onrr.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://onrr.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://onrr.gov/sitemap.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"onhir.gov","name":"Department of the Interior","redirect":"https://www.onhir.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":403,"redirect":"http://onhir.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.onhir.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://onhir.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"nps.gov","name":"Department of the Interior","redirect":"https://nps.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://nps.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"nifc.gov","name":"Department of the Interior","redirect":"https://www.nifc.gov/robots.txt","sitemap":"https://www.nifc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.nifc.gov","valid":false,"allowed":false,"sitemap":null},{"time":1743042289493,"status":200,"redirect":"https://www.nifc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":503,"url":"nfpors.gov","name":"Department of the Interior","redirect":"https://www.nfpors.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://nfpors.gov/robots.txt","valid":true,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://nfpors.gov/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":200,"redirect":"https://nfpors.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"nemi.gov","name":"Department of the Interior","redirect":"https://www.nemi.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.nemi.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"nbc.gov","name":"Department of the Interior","redirect":"http://www.nbc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":301,"url":"nationalmap.gov","name":"Department of the Interior","redirect":"https://www.nationalmap.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1730174316792,"status":301,"redirect":"https://www.nationalmap.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":500,"redirect":"http://www.nationalmap.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"mtbs.gov","name":"Department of the Interior","redirect":"https://www.mtbs.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://mtbs.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://mtbs.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"mrlc.gov","name":"Department of the Interior","redirect":"https://www.mrlc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.mrlc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://www.mrlc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"mms.gov","name":"Department of the Interior","redirect":"http://www.mms.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"mitigationcommission.gov","name":"Department of the Interior","redirect":"http://www.mitigationcommission.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"lmvsci.gov","name":"Department of the Interior","redirect":"http://www.lmvsci.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"lcrmscp.gov","name":"Department of the Interior","redirect":"http://www.lcrmscp.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"lca.gov","name":"Department of the Interior","redirect":"http://www.lca.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"landimaging.gov","name":"Department of the Interior","redirect":"http://www.landimaging.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":503,"url":"landfire.gov","name":"Department of the Interior","redirect":"http://www.landfire.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://landfire.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://landfire.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"lacoast.gov","name":"Department of the Interior","redirect":"https://www.lacoast.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://lacoast.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1740965653672,"status":200,"redirect":"https://www.lacoast.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":404,"url":"jem.gov","name":"Department of the Interior","redirect":"https://www.jem.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://jem.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":500,"url":"invasivespecies.gov","name":"Department of the Interior","redirect":"https://www.invasivespecies.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":522,"redirect":"http://invasivespecies.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":500,"redirect":"https://www.invasivespecies.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1734148194754,"status":500,"redirect":"https://www.doi.gov/invasivespecies/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":500,"url":"interior.gov","name":"Department of the Interior","redirect":"https://www.interior.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":522,"redirect":"http://interior.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":500,"redirect":"https://interior.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1734148194754,"status":500,"redirect":"https://www.doi.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"indianaffairs.gov","name":"Department of the Interior","redirect":"https://www.bia.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.bia.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"iat.gov","name":"Department of the Interior","redirect":"https://www.iat.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1743826667324,"status":500,"redirect":"http://www.iat.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"geoplatform.gov","name":"Department of the Interior","redirect":"https://geoplatform.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://geoplatform.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"gcmrc.gov","name":"Department of the Interior","redirect":"https://www.usgs.gov/centers/sbsc/gcmrchttps:/www.usgs.gov/centers/sbsc/gcmrc","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.usgs.gov/centers/sbsc/gcmrchttps:/www.usgs.gov/centers/sbsc/gcmrc","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://www.usgs.gov/centers/sbsc/gcmrchttps:/www.usgs.gov/centers/sbsc/gcmrc","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":500,"redirect":"https://www.gcmrc.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fws.gov","name":"Department of the Interior","redirect":"https://www.fws.gov/robots.txt","sitemap":"https://www.fws.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://fws.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1743042157917,"status":200,"redirect":"https://www.fws.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"firescience.gov","name":"Department of the Interior","redirect":"https://firescience.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.firescience.gov","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":404,"redirect":"https://firescience.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":200,"redirect":"https://firescience.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1742146243896,"status":404,"redirect":"https://firescience.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"firenet.gov","name":"Department of the Interior","redirect":"https://firenet.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1739509031548,"status":500,"redirect":"http://www.firenet.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"fireleadership.gov","name":"Department of the Interior","redirect":"http://www.fireleadership.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"firecode.gov","name":"Department of the Interior","redirect":"http://www.firecode.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"ficor.gov","name":"Department of the Interior","redirect":"https://ficor.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":403,"redirect":"http://www.ficor.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"fgdc.gov","name":"Department of the Interior","redirect":"https://www.fgdc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://fgdc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1740965653672,"status":200,"redirect":"https://fgdc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1742146243896,"status":500,"redirect":"https://www.fgdc.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://fgdc.gov/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"fcg.gov","name":"Department of the Interior","redirect":"http://www.fcg.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"everykidoutdoors.gov","name":"Department of the Interior","redirect":"http://www.everykidoutdoors.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"evergladesrestoration.gov","name":"Department of the Interior","redirect":"https://www.evergladesrestoration.gov/robots.txt","sitemap":"https://www.evergladesrestoration.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://evergladesrestoration.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.evergladesrestoration.gov/sitemap.xml"}]},{"status":200,"url":"earthquake.gov","name":"Department of the Interior","redirect":"https://www.usgs.gov/programs/earthquake-hazards","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usgs.gov/programs/earthquake-hazards","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usgs.gov/programs/earthquake-hazards","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://earthquake.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"doioig.gov","name":"Department of the Interior","redirect":"https://www.doioig.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.doioig.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"doi.gov","name":"Department of the Interior","redirect":"https://www.doi.gov/robots.txt","sitemap":"https://www.doi.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.doi.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.doi.gov/sitemap.xml"},{"time":1730174316792,"status":200,"redirect":"https://www.doi.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.doi.gov/sitemap.xml","source":"robots.txt"},{"time":1734148194754,"status":500,"redirect":"https://www.doi.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"cupcao.gov","name":"Department of the Interior","redirect":"http://www.cupcao.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"criticalminerals.gov","name":"Department of the Interior","redirect":"https://www.criticalminerals.gov/robots.txt","sitemap":"https://www.criticalminerals.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.criticalminerals.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"coralreef.gov","name":"Department of the Interior","redirect":"https://coralreef.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://coralreef.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.coralreef.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"conservation.gov","name":"Department of the Interior","redirect":"https://www.conservation.gov/robots.txt","sitemap":"https://www.conservation.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":522,"redirect":"http://conservation.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":500,"url":"btfa.gov","name":"Department of the Interior","redirect":"https://www.btfa.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":522,"redirect":"http://btfa.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":500,"redirect":"https://btfa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1734148194754,"status":500,"redirect":"https://www.doi.gov/ost","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"bsee.gov","name":"Department of the Interior","redirect":"https://www.bsee.gov/robots.txt","sitemap":"https://www.bsee.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.bsee.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.bsee.gov/sitemap.xml"}]},{"status":500,"url":"bor.gov","name":"Department of the Interior","redirect":"http://www.bor.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"boemre.gov","name":"Department of the Interior","redirect":"http://www.boemre.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"boem.gov","name":"Department of the Interior","redirect":"https://www.boem.gov/robots.txt","sitemap":"https://www.boem.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.boem.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"blm.gov","name":"Department of the Interior","redirect":"https://www.blm.gov/robots.txt","sitemap":"https://www.blm.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.blm.gov","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.blm.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"bia.gov","name":"Department of the Interior","redirect":"https://www.bia.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.bia.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"anstaskforce.gov","name":"Department of the Interior","redirect":"http://www.anstaskforce.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://www.fws.gov/program/aquatic-nuisance-species-task-force/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"acwi.gov","name":"Department of the Interior","redirect":"https://www.acwi.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.usgs.gov/mission-areas/water-resources/advisory-committee-water-information","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usgs.gov/mission-areas/water-resources/advisory-committee-water-information","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://acwi.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":500,"redirect":"http://www.acwi.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.usgs.gov/mission-areas/water-resources/advisory-committee-water-information","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":500,"redirect":"https://acwi.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":200,"redirect":"https://www.usgs.gov/mission-areas/water-resources/advisory-committee-water-information","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"vets.gov","name":"Department of Veterans Affairs","redirect":"https://www.va.gov/robots.txt","sitemap":"https://www.va.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.va.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.va.gov/sitemap.xml"}]},{"status":200,"url":"vaoig.gov","name":"Department of Veterans Affairs","redirect":"https://www.vaoig.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.vaoig.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"va.gov","name":"Department of Veterans Affairs","redirect":"https://www.va.gov/robots.txt","sitemap":"https://www.va.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.va.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.va.gov/sitemap.xml"}]},{"status":404,"url":"reach.gov","name":"Department of Veterans Affairs","redirect":"https://www.reach.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.va.gov/REACH/default.asp","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"ehr.gov","name":"Department of Veterans Affairs","redirect":"http://www.ehr.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"transportation.gov","name":"Department of Transportation","redirect":"https://www.transportation.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.transportation.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"trafficsafetymarketing.gov","name":"Department of Transportation","redirect":"https://www.trafficsafetymarketing.gov/robots.txt","sitemap":"https://www.trafficsafetymarketing.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.trafficsafetymarketing.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.trafficsafetymarketing.gov/sitemap.xml"}]},{"status":200,"url":"sharetheroadsafely.gov","name":"Department of Transportation","redirect":"https://www.fmcsa.dot.gov/ourroads","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.fmcsa.dot.gov/ourroads","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.sharetheroadsafely.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"safertruck.gov","name":"Department of Transportation","redirect":"https://www.nhtsa.gov/safertruck?redirect-safertruck-sitewide","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.nhtsa.gov/safertruck?redirect-safertruck-sitewide","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.safertruck.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"safercar.gov","name":"Department of Transportation","redirect":"https://www.nhtsa.gov/campaign/safercargov?redirect-safercar-sitewide","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.nhtsa.gov/campaign/safercargov?redirect-safercar-sitewide","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.safercar.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"safeocs.gov","name":"Department of Transportation","redirect":"https://safeocs.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://safeocs.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"safecar.gov","name":"Department of Transportation","redirect":"http://www.safecar.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"protectyourmove.gov","name":"Department of Transportation","redirect":"https://www.fmcsa.dot.gov/protect-your-move","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.fmcsa.dot.gov/protect-your-move","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.protectyourmove.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nhtsa.gov","name":"Department of Transportation","redirect":"https://www.nhtsa.gov/robots.txt","sitemap":"https://www.nhtsa.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nhtsa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.nhtsa.gov/sitemap.xml"}]},{"status":500,"url":"jccbi.gov","name":"Department of Transportation","redirect":"http://www.jccbi.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"flyhealthy.gov","name":"Department of Transportation","redirect":"http://www.flyhealthy.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.flyhealthy.gov/flyhealthy","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://www.transportation.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"flightrights.gov","name":"Department of Transportation","redirect":"https://www.transportation.gov/airconsumer/airline-customer-service-dashboard","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.transportation.gov/airconsumer/airline-customer-service-dashboard","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.transportation.gov/airconsumer/airline-customer-service-dashboard","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://flightrights.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"faasafety.gov","name":"Department of Transportation","redirect":"https://www.faasafety.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://www.faasafety.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"faa.gov","name":"Department of Transportation","redirect":"https://www.faa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"esc.gov","name":"Department of Transportation","redirect":"https://www.esc.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ems.gov","name":"Department of Transportation","redirect":"https://www.ems.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"dotideahub.gov","name":"Department of Transportation","redirect":"http://www.dotideahub.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"dot.gov","name":"Department of Transportation","redirect":"https://www.transportation.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.transportation.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.dot.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"distraction.gov","name":"Department of Transportation","redirect":"http://www.distraction.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"distracteddriving.gov","name":"Department of Transportation","redirect":"https://www.nhtsa.gov/risky-driving/distracted-driving","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nhtsa.gov/risky-driving/distracted-driving","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.distracteddriving.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.distracteddriving.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"bts.gov","name":"Department of Transportation","redirect":"https://www.bts.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"911.gov","name":"Department of Transportation","redirect":"https://www.911.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"usmission.gov","name":"Department of State","redirect":"https://www.usmission.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.state.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"usembassy.gov","name":"Department of State","redirect":"https://www.usembassy.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.usembassy.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"usdoscloud.gov","name":"Department of State","redirect":"http://www.usdoscloud.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"usconsulate.gov","name":"Department of State","redirect":"http://www.usconsulate.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.state.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"usaseanconnect.gov","name":"Department of State","redirect":"http://www.usaseanconnect.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"supportfan.gov","name":"Department of State","redirect":"http://www.supportfan.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"stateoig.gov","name":"Department of State","redirect":"https://www.stateoig.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"state.gov","name":"Department of State","redirect":"https://www.state.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"securitytestfan.gov","name":"Department of State","redirect":"http://www.securitytestfan.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"preprodfan.gov","name":"Department of State","redirect":"http://www.preprodfan.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"pepfar.gov","name":"Department of State","redirect":"https://www.pepfar.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"osac.gov","name":"Department of State","redirect":"https://www.osac.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"http://www.osac.gov/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"}]},{"status":500,"url":"ibwc.gov","name":"Department of State","redirect":"http://www.ibwc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.ibwc.gov","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://www.ibwc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"iawg.gov","name":"Department of State","redirect":"http://www.iawg.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.iawg.gov","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"https://iawg.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://iawg.gov/wp-sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"fsgb.gov","name":"Department of State","redirect":"https://www.fsgb.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"fan.gov","name":"Department of State","redirect":"https://fan.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://fan.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"devtestfan1.gov","name":"Department of State","redirect":"http://www.devtestfan1.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"america.gov","name":"Department of State","redirect":"http://america.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"youthrules.gov","name":"Department of Labor","redirect":"https://www.dol.gov/agencies/whd/youthrules","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dol.gov/agencies/whd/youthrules","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dol.gov/agencies/whd/youthrules","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.youthrules.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"wrp.gov","name":"Department of Labor","redirect":"https://www.wrp.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.wrp.gov/wrp","valid":false,"allowed":false,"sitemap":null}]},{"status":503,"url":"worker.gov","name":"Department of Labor","redirect":"https://www.worker.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.worker.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.worker.gov/sitemap_index.xml"},{"time":1739509031548,"status":200,"redirect":"https://www.worker.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.worker.gov/sitemap_index.xml","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"workcenter.gov","name":"Department of Labor","redirect":"https://www.workcenter.gov/robots.txt","sitemap":"https://www.workcenter.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.workcenter.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.workcenter.gov/sitemap_index.xml"},{"time":1739509031548,"status":200,"redirect":"https://www.workcenter.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.workcenter.gov/sitemap_index.xml","source":"robots.txt"},{"time":1743826667324,"status":503,"redirect":"https://www.workcenter.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"whistleblowers.gov","name":"Department of Labor","redirect":"https://www.whistleblowers.gov/robots.txt","sitemap":"https://www.whistleblowers.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.whistleblowers.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1729037740707,"status":200,"redirect":"https://www.whistleblowers.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.whistleblowers.gov/sitemap.xml","source":"robots.txt"},{"time":1730174316792,"status":200,"redirect":"https://www.whistleblowers.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.whistleblowers.gov/sites/default/files/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"veterans.gov","name":"Department of Labor","redirect":"https://www.dol.gov/agencies/vets?utm_source=veterans_gov&utm_medium=redirect&utm_campaign=veterans_gov_redirect","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dol.gov/agencies/vets?utm_source=veterans_gov&utm_medium=redirect&utm_campaign=veterans_gov_redirect","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"http://www.veterans.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"unionreports.gov","name":"Department of Labor","redirect":"https://www.dol.gov/agencies/olms/public-disclosure-room#union","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1737430190371,"status":500,"redirect":"https://olmsapps.dol.gov/olpdr/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"unemployment.gov","name":"Department of Labor","redirect":"https://www.dol.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.dol.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"ui.gov","name":"Department of Labor","redirect":"https://www.dol.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.dol.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"trainingproviderresults.gov","name":"Department of Labor","redirect":"https://www.trainingproviderresults.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.trainingproviderresults.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"trabajadormigrante.gov","name":"Department of Labor","redirect":"https://www.dol.gov/general/trabajadormigrante","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dol.gov/general/trabajadormigrante","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dol.gov/general/trabajadormigrante","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.trabajadormigrante.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"persuader-reports.gov","name":"Department of Labor","redirect":"https://oprs.dol.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"osha.gov","name":"Department of Labor","redirect":"https://www.osha.gov/robots.txt","sitemap":"https://www.osha.gov/sites/default/files/sitemap-index.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.osha.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.osha.gov/sites/default/files/sitemap-index.xml"}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"mshastanddown.gov","name":"Department of Labor","redirect":"https://www.msha.gov/stand-down-save-lives","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.msha.gov/stand-down-save-lives","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.msha.gov/stand-down-save-lives","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.mshastanddown.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"msha.gov","name":"Department of Labor","redirect":"https://www.msha.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"migrantworker.gov","name":"Department of Labor","redirect":"https://www.dol.gov/general/migrantworker","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dol.gov/general/migrantworker","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dol.gov/general/migrantworker","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.migrantworker.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"labor.gov","name":"Department of Labor","redirect":"http://www.labor.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"jobcorps.gov","name":"Department of Labor","redirect":"https://www.jobcorps.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"hirevets.gov","name":"Department of Labor","redirect":"https://www.hirevets.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.hirevets.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"govloans.gov","name":"Department of Labor","redirect":"https://www.usa.gov/government-loan?utm_source=usa_benefits-gov&utm_medium=redirect&utm_campaign=redirect_govloans-gov","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.govloans.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.govloans.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.usa.gov/government-loan?utm_source=usa_benefits-gov&utm_medium=redirect&utm_campaign=redirect_govloans-gov","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://govloans.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":403,"url":"goodjobs.gov","name":"Department of Labor","redirect":"https://www.dol.gov/general/good-jobs","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dol.gov/general/good-jobs","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dol.gov/general/good-jobs","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.goodjobs.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":200,"redirect":"https://www.dol.gov/general/good-jobs","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":503,"url":"employer.gov","name":"Department of Labor","redirect":"https://www.employer.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.employer.gov","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.employer.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.employer.gov/sitemap_index.xml","source":"robots.txt"}]},{"status":200,"url":"doleta.gov","name":"Department of Labor","redirect":"https://www.dol.gov/agencies/eta","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dol.gov/agencies/eta","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.doleta.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"dol.gov","name":"Department of Labor","redirect":"https://www.dol.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.dol.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"dol-esa.gov","name":"Department of Labor","redirect":"http://www.dol-esa.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"bls.gov","name":"Department of Labor","redirect":"https://www.bls.gov/robots.txt","sitemap":"https://www.bls.gov/sitemap.xml","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"benefits.gov","name":"Department of Labor","redirect":"https://www.usa.gov/benefit-finder?utm_source=usa_benefits-gov&utm_medium=redirect&utm_campaign=redirect_benefits-gov&modal=b-welcome-1899","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.benefits.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.usa.gov/benefit-finder?utm_source=usa_benefits-gov&utm_medium=redirect&utm_campaign=redirect_benefits-gov&modal=b-welcome-1899","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://benefits.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"apprenticeships.gov","name":"Department of Labor","redirect":"https://www.apprenticeship.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.apprenticeship.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"apprenticeship.gov","name":"Department of Labor","redirect":"https://www.apprenticeship.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.apprenticeship.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"vehiclehistory.gov","name":"Department of Justice","redirect":"https://vehiclehistory.bja.ojp.gov/redirect-legacy/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://vehiclehistory.bja.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://vehiclehistory.bja.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.vehiclehistory.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"vcf.gov","name":"Department of Justice","redirect":"https://www.vcf.gov/robots.txt","sitemap":"https://www.vcf.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.vcf.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.vcf.gov/sitemap.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"usmarshals.gov","name":"Department of Justice","redirect":"https://www.usmarshals.gov/robots.txt","sitemap":"https://www.usmarshals.gov/sitemap.xml","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"userra.gov","name":"Department of Justice","redirect":"https://www.justice.gov/servicemembersrobots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.justice.gov/servicemembersrobots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://www.justice.gov/servicemembersrobots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.userra.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"usdoj.gov","name":"Department of Justice","redirect":"https://www.justice.gov/robots.txt","sitemap":"https://www.justice.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1730174316792,"status":500,"redirect":"https://www.usdoj.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":200,"redirect":"https://www.justice.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.justice.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":500,"redirect":"http://www.usdoj.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":302,"url":"unicor.gov","name":"Department of Justice","redirect":"https://www.unicor.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://unicor.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1740965653672,"status":302,"redirect":"https://www.unicor.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":500,"redirect":"http://www.unicor.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"ucrdatatool.gov","name":"Department of Justice","redirect":"http://www.ucrdatatool.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"tsc.gov","name":"Department of Justice","redirect":"https://www.fbi.gov/investigate/terrorism/tscrobots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.fbi.gov/investigate/terrorism/tsc","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":503,"redirect":"https://www.tsc.gov//","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://www.fbi.gov/investigate/terrorism/tscrobots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1739509031548,"status":503,"redirect":"http://www.tsc.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":500,"redirect":"https://www.tsc.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"tribaljusticeandsafety.gov","name":"Department of Justice","redirect":"https://www.justice.gov/tribal/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.justice.gov/tribal/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://www.justice.gov/tribal/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.tribaljusticeandsafety.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"smart.gov","name":"Department of Justice","redirect":"https://smart.ojp.gov/redirect-legacy/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://smart.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://smart.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.smart.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"servicemembers.gov","name":"Department of Justice","redirect":"https://www.justice.gov/servicemembersrobots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.justice.gov/servicemembersrobots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://www.justice.gov/servicemembersrobots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.servicemembers.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"seized.gov","name":"Department of Justice","redirect":"http://www.seized.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"scra.gov","name":"Department of Justice","redirect":"https://www.justice.gov/servicemembersrobots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.justice.gov/servicemembersrobots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://www.justice.gov/servicemembersrobots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.scra.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"reentry.gov","name":"Department of Justice","redirect":"http://www.reentry.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"rcfl.gov","name":"Department of Justice","redirect":"https://www.rcfl.gov/robots.txt","sitemap":"https://www.rcfl.gov/sitemap.xml.gz","history":[{"time":1726617445612,"status":200,"redirect":"https://www.rcfl.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"psob.gov","name":"Department of Justice","redirect":"https://psob.bja.ojp.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://psob.bja.ojp.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://psob.bja.ojp.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.psob.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"projectsafeneighborhoods.gov","name":"Department of Justice","redirect":"https://www.justice.gov/psn","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.justice.gov/psn","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.justice.gov/psn","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://projectsafeneighborhoods.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":200,"redirect":"https://www.justice.gov/psn","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.projectsafeneighborhoods.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"projectsafechildhood.gov","name":"Department of Justice","redirect":"https://www.justice.gov/psc/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.justice.gov/psc/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://www.justice.gov/psc/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.projectsafechildhood.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ovcttac.gov","name":"Department of Justice","redirect":"https://www.ovcttac.gov/robots.txt","sitemap":"https://www.ovcttac.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ovcttac.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ovcttac.gov/views/sitemap.xml"},{"time":1740965653672,"status":200,"redirect":"https://www.ovcttac.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ovcttac.gov/views/sitemap.xml","source":"robots.txt"}]},{"status":404,"url":"ovc.gov","name":"Department of Justice","redirect":"https://ovc.ojp.gov/redirect-legacy/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://ovc.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://ovc.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.ovc.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ojp.gov","name":"Department of Justice","redirect":"https://www.ojp.gov/robots.txt","sitemap":"https://www.ojp.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://ojp.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ojp.gov/sitemap.xml"},{"time":1734148194754,"status":200,"redirect":"https://ojp.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ojp.gov/sitemap.xml","source":"robots.txt"},{"time":1740965653672,"status":200,"redirect":"https://ojp.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://ojp.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":200,"redirect":"https://www.ojp.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ojp.gov/sitemap.xml","source":"robots.txt"},{"time":1743826667324,"status":200,"redirect":"https://ojp.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://ojp.gov/sitemap.xml","source":"robots.txt"}]},{"status":404,"url":"ojjdp.gov","name":"Department of Justice","redirect":"https://ojjdp.ojp.gov/redirect-legacy/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://ojjdp.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://ojjdp.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.ojjdp.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"nvtc.gov","name":"Department of Justice","redirect":"https://www.fbi.gov/nvtc","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.fbi.gov/about/leadership-and-structure/intelligence/national-virtual-translation-center","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.fbi.gov/about/leadership-and-structure/intelligence/national-virtual-translation-center","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1729037740707,"status":301,"redirect":"https://www.nvtc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.nvtc.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://www.fbi.gov/about/leadership-and-structure/intelligence/national-virtual-translation-center","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1740965653672,"status":403,"redirect":"https://www.fbi.gov/nvtc","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":200,"redirect":"https://www.fbi.gov/about/leadership-and-structure/intelligence/national-virtual-translation-center","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"nsopw.gov","name":"Department of Justice","redirect":"https://www.nsopw.gov/robots.txt","sitemap":"https://www.nsopw.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nsopw.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.nsopw.gov/sitemap.xml"}]},{"status":500,"url":"nsopr.gov","name":"Department of Justice","redirect":"http://www.nsopr.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"nmvtis.gov","name":"Department of Justice","redirect":"https://vehiclehistory.bja.ojp.gov/robots.txt","sitemap":"https://vehiclehistory.bja.ojp.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"https://vehiclehistory.bja.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":200,"redirect":"https://vehiclehistory.bja.ojp.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://vehiclehistory.bja.ojp.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":500,"redirect":"http://www.nmvtis.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"nij.gov","name":"Department of Justice","redirect":"https://nij.ojp.gov/redirect-legacy/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://nij.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://nij.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.nij.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"nicsezcheckfbi.gov","name":"Department of Justice","redirect":"https://nicsezcheckfbi.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://nicsezcheckfbi.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"nicic.gov","name":"Department of Justice","redirect":"https://nicic.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://nicic.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"ncjrs.gov","name":"Department of Justice","redirect":"https://www.ncjrs.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.ncjrs.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":200,"redirect":"https://www.ncjrs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ncjrs.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":500,"redirect":"http://www.ncjrs.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ncirc.gov","name":"Department of Justice","redirect":"https://ncirc.bja.ojp.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://ncirc.bja.ojp.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://ncirc.bja.ojp.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.ncirc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"nationalgangcenter.gov","name":"Department of Justice","redirect":"https://nationalgangcenter.ojp.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://nationalgangcenter.ojp.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://nationalgangcenter.ojp.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://nationalgangcenter.ojp.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"namus.gov","name":"Department of Justice","redirect":"https://www.namus.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.namus.gov/robots.txt","valid":true,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.namus.gov/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"},{"time":1740965653672,"status":200,"redirect":"https://www.namus.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1742146243896,"status":500,"redirect":"http://www.namus.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"medalofvalor.gov","name":"Department of Justice","redirect":"https://www.medalofvalor.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.medalofvalor.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://www.medalofvalor.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.medalofvalor.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"malwareinvestigator.gov","name":"Department of Justice","redirect":"http://www.malwareinvestigator.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"lep.gov","name":"Department of Justice","redirect":"https://www.lep.gov/robots.txt","sitemap":"https://www.lep.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.lep.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.lep.gov/sitemap.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.lep.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"leo.gov","name":"Department of Justice","redirect":"http://www.leo.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"learndoj.gov","name":"Department of Justice","redirect":"https://www.justice.gov/jmd/learndoj","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.justice.gov/jmd/learndoj","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.learndoj.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://learndoj.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"learnatf.gov","name":"Department of Justice","redirect":"https://learnatf.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://learnatf.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.learnatf.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"juvenilecouncil.gov","name":"Department of Justice","redirect":"https://juvenilecouncil.ojp.gov/redirect-legacy/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://juvenilecouncil.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.juvenilecouncil.gov","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://juvenilecouncil.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.juvenilecouncil.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"justthinktwice.gov","name":"Department of Justice","redirect":"https://www.justthinktwice.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.justthinktwice.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"justice.gov","name":"Department of Justice","redirect":"https://www.justice.gov/robots.txt","sitemap":"https://www.justice.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.justice.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.justice.gov/sitemap.xml"}]},{"status":500,"url":"jcode.gov","name":"Department of Justice","redirect":"http://www.jcode.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"iprcenter.gov","name":"Department of Justice","redirect":"https://www.iprcenter.gov/robots.txt","sitemap":"https://www.iprcenter.gov/sitemap.xml.gz","history":[{"time":1726617445612,"status":200,"redirect":"https://www.iprcenter.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.iprcenter.gov/sitemap.xml.gz"}]},{"status":404,"url":"interpol.gov","name":"Department of Justice","redirect":"https://www.justice.gov/interpol-washington/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.justice.gov/interpol-washington/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://www.justice.gov/interpol-washington/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.interpol.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ic3.gov","name":"Department of Justice","redirect":"https://www.ic3.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ic3.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"getsmartaboutdrugs.gov","name":"Department of Justice","redirect":"https://www.getsmartaboutdrugs.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.getsmartaboutdrugs.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"fpi.gov","name":"Department of Justice","redirect":"http://www.fpi.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"forfeiture.gov","name":"Department of Justice","redirect":"https://www.forfeiture.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":403,"redirect":"https://www.forfeiture.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"foia.gov","name":"Department of Justice","redirect":"https://www.foia.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.foia.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"firstfreedom.gov","name":"Department of Justice","redirect":"https://www.justice.gov/crt/combating-religious-discrimination-and-protecting-religious-freedom-12robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.justice.gov/crt/combating-religious-discrimination-and-protecting-religious-freedom-12robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://www.justice.gov/crt/combating-religious-discrimination-and-protecting-religious-freedom-12robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.firstfreedom.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"fbilab.gov","name":"Department of Justice","redirect":"http://www.fbilab.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"fbijobs.gov","name":"Department of Justice","redirect":"https://www.fbijobs.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"fbihr.gov","name":"Department of Justice","redirect":"http://www.fbihr.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"fbi.gov","name":"Department of Justice","redirect":"https://www.fbi.gov/robots.txt","sitemap":"https://www.fbi.gov/sitemap.xml.gz","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fbi.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fbi.gov/sitemap.xml.gz"},{"time":1728949876846,"status":500,"redirect":"http://www.fbi.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"fara.gov","name":"Department of Justice","redirect":"https://www.fara.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"epic.gov","name":"Department of Justice","redirect":"https://www.dea.gov/what-we-do/law-enforcement/epic","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dea.gov/what-we-do/law-enforcement/epic","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.epic.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.epic.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":404,"url":"elderjustice.gov","name":"Department of Justice","redirect":"https://www.justice.gov/elderjustice/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.justice.gov/elderjustice/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"dsac.gov","name":"Department of Justice","redirect":"https://www.dsac.gov/robots.txt","sitemap":"https://www.dsac.gov/sitemap.xml.gz","history":[{"time":1726617445612,"status":200,"redirect":"https://www.dsac.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"doj.gov","name":"Department of Justice","redirect":"http://www.doj.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"deaecom.gov","name":"Department of Justice","redirect":"https://deaecom.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://deaecom.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"dea.gov","name":"Department of Justice","redirect":"https://www.dea.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dea.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"cybercrime.gov","name":"Department of Justice","redirect":"https://www.justice.gov/criminal/criminal-ccips","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.justice.gov/criminal/criminal-ccips","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.justice.gov/criminal/criminal-ccips","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.cybercrime.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":200,"redirect":"https://www.justice.gov/criminal/criminal-ccips","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.cybercrime.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"crimevictims.gov","name":"Department of Justice","redirect":"https://ovc.ojp.gov/redirect-legacy/crimevictims/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://ovc.ojp.gov/redirect-legacy/crimevictims/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://ovc.ojp.gov/redirect-legacy/crimevictims/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.crimevictims.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"crimesolutions.gov","name":"Department of Justice","redirect":"https://crimesolutions.ojp.gov/redirect-legacy/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://crimesolutions.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://crimesolutions.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.crimesolutions.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"cjis.gov","name":"Department of Justice","redirect":"http://www.cjis.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"campusdrugprevention.gov","name":"Department of Justice","redirect":"https://www.campusdrugprevention.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.campusdrugprevention.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"bop.gov","name":"Department of Justice","redirect":"https://www.bop.gov/robots.txt","sitemap":"http://www.bop.gov/PublicInfo/execute/foiadoc?todo=query&output=xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"bjs.gov","name":"Department of Justice","redirect":"https://bjs.ojp.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://bjs.ojp.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://bjs.ojp.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://bjs.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":200,"redirect":"https://bjs.ojp.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.bjs.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"bja.gov","name":"Department of Justice","redirect":"http://www.bja.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"biometriccoe.gov","name":"Department of Justice","redirect":"https://www.fbi.gov/about-us/cjis/fingerprints_biometrics/biometric-center-of-excellence/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://fbibiospecs.fbi.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":403,"redirect":"https://www.fbi.gov/about-us/cjis/fingerprints_biometrics/biometric-center-of-excellence/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.biometriccoe.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1743826667324,"status":200,"redirect":"https://www.fbi.gov/","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"bats.gov","name":"Department of Justice","redirect":"https://bats.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://live.bats.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"atfonline.gov","name":"Department of Justice","redirect":"http://www.atfonline.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"atf.gov","name":"Department of Justice","redirect":"https://www.atf.gov/robots.txt","sitemap":"http://www.atf.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"amberalert.gov","name":"Department of Justice","redirect":"https://amberalert.ojp.gov/redirect-legacy/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://amberalert.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://amberalert.ojp.gov/redirect-legacy/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.amberalert.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ada.gov","name":"Department of Justice","redirect":"https://www.ada.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ada.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ada.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://www.ada.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ada.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"opportunityzones.gov","name":"Department of Housing and Urban Development","redirect":"https://opportunityzones.hud.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://opportunityzones.hud.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"nls.gov","name":"Department of Housing and Urban Development","redirect":"https://www.hud.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.hud.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.hud.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://nls.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"nhl.gov","name":"Department of Housing and Urban Development","redirect":"https://www.hud.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.hud.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.hud.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.nhl.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"nationalhousinglocator.gov","name":"Department of Housing and Urban Development","redirect":"https://www.hud.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.hud.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.hud.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://nationalhousinglocator.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"nationalhousing.gov","name":"Department of Housing and Urban Development","redirect":"https://www.hud.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.hud.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.hud.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://nationalhousing.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"huduser.gov","name":"Department of Housing and Urban Development","redirect":"https://www.huduser.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.huduser.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"hudoig.gov","name":"Department of Housing and Urban Development","redirect":"https://hudoig.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://hudoig.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":403,"redirect":"https://www.hudoig.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"hudhomestore.gov","name":"Department of Housing and Urban Development","redirect":"https://www.hudhomestore.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":"https://www.hudhomestore.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":500,"redirect":"https://www.hudhomestore.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"hud.gov","name":"Department of Housing and Urban Development","redirect":"https://www.hud.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.hud.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"ginniemae.gov","name":"Department of Housing and Urban Development","redirect":"https://www.ginniemae.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"https://www.ginniemae.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":404,"redirect":"https://www.ginniemae.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"fha.gov","name":"Department of Housing and Urban Development","redirect":"https://www.hud.gov/fha","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.hud.gov/fha","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.hud.gov/fha","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://fha.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"disasterhousing.gov","name":"Department of Housing and Urban Development","redirect":"https://www.hud.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.hud.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.hud.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://disasterhousing.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"usss.gov","name":"Department of Homeland Security","redirect":"https://www.secretservice.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.secretservice.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.secretservice.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.usss.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"uscis.gov","name":"Department of Homeland Security","redirect":"https://www.uscis.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.uscis.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"uscg.gov","name":"Department of Homeland Security","redirect":"http://www.uscg.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"us-cert.gov","name":"Department of Homeland Security","redirect":"https://www.cisa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cisa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"tsa.gov","name":"Department of Homeland Security","redirect":"https://www.tsa.gov/robots.txt","sitemap":"http://www.tsa.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.tsa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"http://www.tsa.gov/sitemap.xml"}]},{"status":200,"url":"together.gov","name":"Department of Homeland Security","redirect":"https://www.together.gov/robots.txt","sitemap":null,"valid":true,"allowed":false,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.together.gov/robots.txt","valid":true,"allowed":false,"sitemap":null}]},{"status":200,"url":"stopransomware.gov","name":"Department of Homeland Security","redirect":"https://www.cisa.gov/stopransomware","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cisa.gov/stopransomware","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cisa.gov/stopransomware","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://www.stopransomware.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"secretservice.gov","name":"Department of Homeland Security","redirect":"https://www.secretservice.gov/sites/default/files/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.secretservice.gov/sites/default/files/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"schoolsafety.gov","name":"Department of Homeland Security","redirect":"https://www.schoolsafety.gov/robots.txt","sitemap":"https://www.schoolsafety.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.schoolsafety.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.schoolsafety.gov/sitemap.xml"}]},{"status":404,"url":"safetyact.gov","name":"Department of Homeland Security","redirect":"https://www.safetyact.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.safetyact.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"readybusiness.gov","name":"Department of Homeland Security","redirect":"https://www.ready.gov/business","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ready.gov/business","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.ready.gov/business","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.readybusiness.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"ready.gov","name":"Department of Homeland Security","redirect":"https://www.ready.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ready.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"preventionresourcefinder.gov","name":"Department of Homeland Security","redirect":"https://www.dhs.gov/prevention","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dhs.gov/prevention","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dhs.gov/prevention","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.preventionresourcefinder.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"power2prevent.gov","name":"Department of Homeland Security","redirect":"http://www.power2prevent.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ns.gov","name":"Department of Homeland Security","redirect":"http://www.ns.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nmsc.gov","name":"Department of Homeland Security","redirect":"http://www.nmsc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"niem.gov","name":"Department of Homeland Security","redirect":"https://www.niem.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.niem.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":403,"url":"nic.gov","name":"Department of Homeland Security","redirect":"http://www.nic.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"listo.gov","name":"Department of Homeland Security","redirect":"https://www.ready.gov/es/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.listo.gov/es/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"know2protect.gov","name":"Department of Homeland Security","redirect":"https://www.dhs.gov/know2protect","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dhs.gov/know2protect","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dhs.gov/know2protect","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://know2protect.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"juntos.gov","name":"Department of Homeland Security","redirect":"https://www.juntos.gov/robots.txt","sitemap":null,"valid":true,"allowed":false,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.juntos.gov/robots.txt","valid":true,"allowed":false,"sitemap":null}]},{"status":200,"url":"ice.gov","name":"Department of Homeland Security","redirect":"https://www.ice.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ice.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"homelandsecurity.gov","name":"Department of Homeland Security","redirect":"https://www.dhs.gov/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dhs.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dhs.gov/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.homelandsecurity.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"globalentry.gov","name":"Department of Homeland Security","redirect":"https://www.cbp.gov/travel/trusted-traveler-programs/global-entry","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cbp.gov/travel/trusted-traveler-programs/global-entry","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cbp.gov/travel/trusted-traveler-programs/global-entry","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://globalentry.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"get.gov","name":"Department of Homeland Security","redirect":"https://get.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://get.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":403,"url":"floodsmart.gov","name":"Department of Homeland Security","redirect":"https://www.floodsmart.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":403,"redirect":"https://www.floodsmart.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fletc.gov","name":"Department of Homeland Security","redirect":"https://www.fletc.gov/robots.txt","sitemap":"https://www.fletc.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fletc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fletc.gov/sitemap.xml"}]},{"status":200,"url":"fleta.gov","name":"Department of Homeland Security","redirect":"https://www.fleta.gov/robots.txt","sitemap":"https://www.fleta.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fleta.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fleta.gov/sitemap.xml"}]},{"status":404,"url":"firstrespondertraining.gov","name":"Department of Homeland Security","redirect":"https://www.firstrespondertraining.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.firstrespondertraining.gov/frts/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fema.gov","name":"Department of Homeland Security","redirect":"https://www.fema.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fema.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"evus.gov","name":"Department of Homeland Security","redirect":"https://www.evus.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.evus.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"everify.gov","name":"Department of Homeland Security","redirect":"https://www.e-verify.gov/sites/default/files/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.e-verify.gov/sites/default/files/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"e-verify.gov","name":"Department of Homeland Security","redirect":"https://www.e-verify.gov/sites/default/files/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.e-verify.gov/sites/default/files/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"dotgov.gov","name":"Department of Homeland Security","redirect":"https://get.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://get.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"dns.gov","name":"Department of Homeland Security","redirect":"http://www.dns.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"disasterassistance.gov","name":"Department of Homeland Security","redirect":"https://www.disasterassistance.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":403,"redirect":"https://disasterassistance.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"dhs.gov","name":"Department of Homeland Security","redirect":"https://www.dhs.gov/robots.txt","sitemap":"https://www.dhs.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.dhs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.dhs.gov/sitemap.xml"}]},{"status":200,"url":"cybersecurity.gov","name":"Department of Homeland Security","redirect":"https://www.cisa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cisa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"cyber.gov","name":"Department of Homeland Security","redirect":"https://www.cisa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cisa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"cpnireporting.gov","name":"Department of Homeland Security","redirect":"http://www.cpnireporting.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1734148194754,"status":500,"redirect":"http://www.cpnireporting.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":404,"redirect":"https://www.cpnireporting.gov/content/disclaimer.xhtml","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":500,"redirect":"http://www.cpnireporting.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://www.cpnireporting.gov/content/disclaimer.xhtml","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"cisa.gov","name":"Department of Homeland Security","redirect":"https://www.cisa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cisa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"cbp.gov","name":"Department of Homeland Security","redirect":"https://www.cbp.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cbp.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"biometrics.gov","name":"Department of Homeland Security","redirect":"https://www.dhs.gov/biometrics","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dhs.gov/biometrics","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dhs.gov/biometrics","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://biometrics.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":403,"url":"youth.gov","name":"Department of Health and Human Services","redirect":"https://youth.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://youth.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://youth.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":500,"redirect":"https://youth.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"womenshealth.gov","name":"Department of Health and Human Services","redirect":"https://womenshealth.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.womenshealth.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"whitehouseconferenceonaging.gov","name":"Department of Health and Human Services","redirect":"https://whitehouseconferenceonaging.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://whitehouseconferenceonaging.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"whaging.gov","name":"Department of Health and Human Services","redirect":"http://www.whaging.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"visforvaccinated.gov","name":"Department of Health and Human Services","redirect":"https://www.hhs.gov/visforvaccinated/index.html","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":404,"redirect":"https://www.hhs.gov/visforvaccinated/index.html","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":301,"redirect":"https://www.visforvaccinated.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"vacunas.gov","name":"Department of Health and Human Services","redirect":"https://www.vaccines.gov/es/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.vacunas.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.vaccines.gov/es/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.vacunas.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"vacuna.gov","name":"Department of Health and Human Services","redirect":"https://www.vaccines.gov/es/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.vacunas.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.vaccines.gov/es/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.vacunas.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"vacines.gov","name":"Department of Health and Human Services","redirect":"https://www.vaccines.gov/en/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.vaccines.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.vaccines.gov/en/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.vaccines.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"vacine.gov","name":"Department of Health and Human Services","redirect":"https://www.vaccines.gov/en/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.vaccines.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.vaccines.gov/en/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.vaccines.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"vaccines.gov","name":"Department of Health and Human Services","redirect":"https://www.vaccines.gov/en/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.vaccines.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.vaccines.gov/en/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.vaccines.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"vaccine.gov","name":"Department of Health and Human Services","redirect":"https://www.vaccines.gov/en/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.vaccines.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.vaccines.gov/en/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.vaccines.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"usphs.gov","name":"Department of Health and Human Services","redirect":"https://www.usphs.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"usbm.gov","name":"Department of Health and Human Services","redirect":"http://www.usbm.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"tox21.gov","name":"Department of Health and Human Services","redirect":"https://tox21.gov/robots.txt","sitemap":"https://tox21.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://tox21.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://tox21.gov/sitemap_index.xml"}]},{"status":500,"url":"tobacco.gov","name":"Department of Health and Human Services","redirect":"https://betobaccofree.hhs.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://betobaccofree.hhs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://betobaccofree.hhs.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://betobaccofree.hhs.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.tobacco.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"https://betobaccofree.hhs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://betobaccofree.hhs.gov/sitemap.xml","source":"robots.txt"}]},{"status":404,"url":"thisfreelife.gov","name":"Department of Health and Human Services","redirect":"https://smokefree.gov/lgbt-and-smokingrobots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://smokefree.gov/lgbt-and-smokingrobots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://smokefree.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.thisfreelife.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":404,"redirect":"https://smokefree.gov/lgbt-and-smokingrobots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":500,"redirect":"https://smokefree.gov/lgbt-and-smoking","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"therealcost.gov","name":"Department of Health and Human Services","redirect":"https://therealcost.betobaccofree.hhs.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://therealcost.betobaccofree.hhs.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://therealcost.betobaccofree.hhs.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1740965653672,"status":500,"redirect":"https://therealcost.betobaccofree.hhs.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"thebraininitiative.gov","name":"Department of Health and Human Services","redirect":"https://braininitiative.nih.govrobots.txt/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://braininitiative.nih.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://braininitiative.nih.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":404,"url":"surgeongeneral.gov","name":"Department of Health and Human Services","redirect":"https://www.hhs.gov/surgeongeneral/not-found/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.hhs.gov/surgeongeneral/not-found/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"stopbullying.gov","name":"Department of Health and Human Services","redirect":"https://www.stopbullying.gov/robots.txt","sitemap":"https://www.stopbullying.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.stopbullying.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.stopbullying.gov/sitemap.xml"}]},{"status":200,"url":"stopalcoholabuse.gov","name":"Department of Health and Human Services","redirect":"https://www.stopalcoholabuse.gov/robots.txt","sitemap":"https://www.stopalcoholabuse.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"smokefree.gov","name":"Department of Health and Human Services","redirect":"https://smokefree.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://smokefree.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://smokefree.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":500,"redirect":"http://www.smokefree.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"simplereport.gov","name":"Department of Health and Human Services","redirect":"https://www.simplereport.gov/robots.txt","sitemap":"https://www.simplereport.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.simplereport.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.simplereport.gov/sitemap.xml"}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"selectagents.gov","name":"Department of Health and Human Services","redirect":"https://selectagents.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://selectagents.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":400,"url":"samhsa.gov","name":"Department of Health and Human Services","redirect":"http://www.samhsa.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"reproductivehealthservices.gov","name":"Department of Health and Human Services","redirect":"https://reproductivehealthservices.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://reproductivehealthservices.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"recoverymonth.gov","name":"Department of Health and Human Services","redirect":"https://www.samhsa.gov/about/digital-toolkits/recovery-month","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.samhsa.gov/recovery-month","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.samhsa.gov/recovery-month","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://www.recoverymonth.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"pubmed.gov","name":"Department of Health and Human Services","redirect":"https://misuse.ncbi.nlm.nih.gov/error/abuse.shtml","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://misuse.ncbi.nlm.nih.gov/error/abuse.shtml","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"https://misuse.ncbi.nlm.nih.gov/error/abuse.shtml","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1739509031548,"status":500,"redirect":"https://pubmed.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"psc.gov","name":"Department of Health and Human Services","redirect":"https://www.hhs.gov/about/agencies/asa/psc/index.html","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.hhs.gov/about/agencies/asa/psc/index.htmlrobots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.hhs.gov/about/agencies/asa/psc/index.html","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://psc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":403,"url":"phe.gov","name":"Department of Health and Human Services","redirect":"https://aspr.hhs.gov/Pages/Home.aspx","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.phe.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://prod-phe.hhs.gov:443/sitemap.xml"},{"time":1734148194754,"status":200,"redirect":"http://www.phe.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://prod-phe.hhs.gov:443/sitemap.xml","source":"robots.txt"}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"pandemicflu.gov","name":"Department of Health and Human Services","redirect":"https://www.cdc.gov/pandemic-flu/index.html","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cdc.gov/flu/pandemic-resources/index.htm","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cdc.gov/pandemic-flu/index.html","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.pandemicflu.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"organdonor.gov","name":"Department of Health and Human Services","redirect":"https://www.organdonor.gov/robots.txt","sitemap":"https://www.organdonor.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.organdonor.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.organdonor.gov/sitemap.xml"}]},{"status":404,"url":"opioids.gov","name":"Department of Health and Human Services","redirect":"https://www.hhs.gov/opioids/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.hhs.gov/opioids/index.html","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"nnlm.gov","name":"Department of Health and Human Services","redirect":"https://www.nnlm.gov/robots.txt","sitemap":"https://nnlm.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nnlm.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://nnlm.gov/sitemap.xml"}]},{"status":200,"url":"nlm.gov","name":"Department of Health and Human Services","redirect":"https://www.nlm.nih.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nlm.nih.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"niosh.gov","name":"Department of Health and Human Services","redirect":"https://www.niosh.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nih.gov","name":"Department of Health and Human Services","redirect":"https://www.nih.gov/robots.txt","sitemap":"https://www.nih.gov/sitemap.xml","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"nextlegends.gov","name":"Department of Health and Human Services","redirect":"https://therealcost.betobaccofree.hhs.gov/nextlegends","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://therealcost.betobaccofree.hhs.gov/nextlegends","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://therealcost.betobaccofree.hhs.gov/nextlegends","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://nextlegends.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"https://therealcost.betobaccofree.hhs.gov/nextlegends","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":500,"redirect":"https://therealcost.betobaccofree.hhs.gov/nextlegends","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"ncifcrf.gov","name":"Department of Health and Human Services","redirect":"http://www.ncifcrf.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"mymedicare.gov","name":"Department of Health and Human Services","redirect":"https://www.medicare.gov/my/home","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.medicare.gov/my/home","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.medicare.gov/my/home","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.mymedicare.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"mesh.gov","name":"Department of Health and Human Services","redirect":"https://meshb.nlm.nih.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://meshb.nlm.nih.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://meshb.nlm.nih.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.mesh.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"mentalhealth.gov","name":"Department of Health and Human Services","redirect":"https://www.samhsa.gov/mental-health","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.samhsa.gov/mental-health","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.samhsa.gov/mental-health","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.mentalhealth.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":200,"redirect":"https://www.samhsa.gov/mental-health","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":200,"redirect":"https://www.samhsa.gov/core/install.php","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"medlineplus.gov","name":"Department of Health and Human Services","redirect":"https://medlineplus.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://medlineplus.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://medlineplus.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://medlineplus.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://medlineplus.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"medicare.gov","name":"Department of Health and Human Services","redirect":"https://www.medicare.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.medicare.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"medicalcountermeasures.gov","name":"Department of Health and Human Services","redirect":"https://medicalcountermeasures.gov/robots.txt","sitemap":"https://www.medicalcountermeasures.gov/xmlsitemap","history":[{"time":1728949876846,"status":500,"redirect":"http://www.medicalcountermeasures.gov","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"https://medicalcountermeasures.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.medicalcountermeasures.gov/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"medicalbillrights.gov","name":"Department of Health and Human Services","redirect":"http://www.medicalbillrights.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"medicaid.gov","name":"Department of Health and Human Services","redirect":"https://www.medicaid.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.medicaid.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"longtermcare.gov","name":"Department of Health and Human Services","redirect":"https://acl.gov/ltc/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":502,"redirect":"http://www.longtermcare.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"insurekidsnow.gov","name":"Department of Health and Human Services","redirect":"https://www.insurekidsnow.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.insurekidsnow.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.insurekidsnow.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"ihs.gov","name":"Department of Health and Human Services","redirect":"https://www.ihs.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ihs.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":503,"url":"idealab.gov","name":"Department of Health and Human Services","redirect":"http://www.idealab.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.hhs.gov/cto/index.html","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":404,"redirect":"https://www.hhs.gov/about/archive/index.htmlrobots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":403,"redirect":"http://www.idealab.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"hrsa.gov","name":"Department of Health and Human Services","redirect":"https://www.hrsa.gov/robots.txt","sitemap":"https://www.hrsa.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"hiv.gov","name":"Department of Health and Human Services","redirect":"https://www.hiv.gov/robots.txt","sitemap":"https://www.hiv.gov/sitemap-index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.hiv.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.hiv.gov/sitemap-index.xml"}]},{"status":500,"url":"hhsops.gov","name":"Department of Health and Human Services","redirect":"http://www.hhsops.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"hhsoig.gov","name":"Department of Health and Human Services","redirect":"http://www.hhsoig.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"hhs.gov","name":"Department of Health and Human Services","redirect":"https://www.hhs.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"https://www.hhs.gov/web/508/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"hearttruth.gov","name":"Department of Health and Human Services","redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.nhlbi.nih.gov/health-topics/education-and-awareness/heart-truth","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://hearttruth.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"healthypeople.gov","name":"Department of Health and Human Services","redirect":"https://odphp.health.gov/healthypeople/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.healthypeople.govhttps//www.healthypeople.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://healthypeople.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1734148194754,"status":200,"redirect":"https://odphp.health.gov/healthypeople","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"healthit.gov","name":"Department of Health and Human Services","redirect":"https://www.healthit.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.healthit.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"healthindicators.gov","name":"Department of Health and Human Services","redirect":"http://www.healthindicators.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"healthfinder.gov","name":"Department of Health and Human Services","redirect":"https://odphp.health.gov/myhealthfinder/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://health.gov/myhealthfinder/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":404,"redirect":"https://health.gov/myhealthfinder/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://www.healthfinder.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"healthdata.gov","name":"Department of Health and Human Services","redirect":"https://healthdata.gov/robots.txt","sitemap":"https://s3.amazonaws.com/sa-socrata-sitemaps-us-east-1-fedramp-prod/sitemaps/sitemap-healthdata.gov.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://healthdata.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://s3.amazonaws.com/sa-socrata-sitemaps-us-east-1-fedramp-prod/sitemaps/sitemap-healthdata.gov.xml"}]},{"status":200,"url":"healthcare.gov","name":"Department of Health and Human Services","redirect":"https://www.healthcare.gov/robots.txt","sitemap":"https://www.healthcare.gov/sitemap-index.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.healthcare.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.healthcare.gov/sitemap-index.xml"}]},{"status":200,"url":"health.gov","name":"Department of Health and Human Services","redirect":"https://health.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://health.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"hc.gov","name":"Department of Health and Human Services","redirect":"http://www.hc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://go.wh.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":403,"url":"guidelines.gov","name":"Department of Health and Human Services","redirect":"https://www.guidelines.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ahrq.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"guideline.gov","name":"Department of Health and Human Services","redirect":"https://www.guideline.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ahrq.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"grantsolutions.gov","name":"Department of Health and Human Services","redirect":"https://www.grantsolutions.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.grantsolutions.gov/robots.txt","valid":true,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.grantsolutions.gov/robots.txt","valid":true,"allowed":false,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"grants.gov","name":"Department of Health and Human Services","redirect":"https://grants.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://grants.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"gmta.gov","name":"Department of Health and Human Services","redirect":"https://gmta.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://gmta.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"globalhealth.gov","name":"Department of Health and Human Services","redirect":"https://www.hhs.gov/about/agencies/oga/index.html","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.hhs.gov/about/agencies/ogarobots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.hhs.gov/about/agencies/oga/index.html","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":404,"redirect":"https://www.hhs.gov/about/agencies/ogarobots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"girlshealth.gov","name":"Department of Health and Human Services","redirect":"https://girlshealth.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.girlshealth.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"genome.gov","name":"Department of Health and Human Services","redirect":"https://www.genome.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.genome.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"fruitsandveggiesmatter.gov","name":"Department of Health and Human Services","redirect":"http://www.fruitsandveggiesmatter.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"freshempire.gov","name":"Department of Health and Human Services","redirect":"https://www.freshempire.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"foodsafety.gov","name":"Department of Health and Human Services","redirect":"https://www.foodsafety.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.foodsafety.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.globalsiteseo.com/foodsafetygov.GSM/foodsafetygov.gsm.index.en_super.xml"},{"time":1742146243896,"status":500,"redirect":"https://www.foodsafety.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://www.foodsafety.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.globalsiteseo.com/foodsafetygov.GSM/foodsafetygov.gsm.index.en_super.xml","source":"robots.txt"}]},{"status":500,"url":"flu.gov","name":"Department of Health and Human Services","redirect":"http://www.flu.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cdc.gov/flu/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.flu.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.flu.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1734148194754,"status":200,"redirect":"https://www.cdc.gov/flu/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":500,"redirect":"https://www.flu.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.cdc.gov/flu/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1740965653672,"status":500,"redirect":"https://www.flu.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://www.cdc.gov/flu/","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"fitness.gov","name":"Department of Health and Human Services","redirect":"https://odphp.health.gov/pcsfn","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.fitness.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.hhs.gov/fitness/","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"findtreatment.gov","name":"Department of Health and Human Services","redirect":"https://findtreatment.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://findtreatment.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.findtreatment.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"findsupport.gov","name":"Department of Health and Human Services","redirect":"https://www.samhsa.gov/find-support/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.samhsa.gov/find-support/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.findsupport.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fda.gov","name":"Department of Health and Human Services","redirect":"https://www.fda.gov/robots.txt","sitemap":"https://www.fda.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.fda.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.fda.gov/sitemap.xml"}]},{"status":403,"url":"fatherhood.gov","name":"Department of Health and Human Services","redirect":"http://www.fatherhood.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"familyplanning.gov","name":"Department of Health and Human Services","redirect":"https://familyplanning.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://reproductivehealthservices.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"everytrycounts.gov","name":"Department of Health and Human Services","redirect":"https://everytrycounts.betobaccofree.hhs.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://everytrycounts.betobaccofree.hhs.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://everytrycounts.betobaccofree.hhs.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.everytrycounts.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"https://everytrycounts.betobaccofree.hhs.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1740965653672,"status":500,"redirect":"https://everytrycounts.betobaccofree.hhs.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"encuentraapoyo.gov","name":"Department of Health and Human Services","redirect":"https://www.samhsa.gov/encuentra-apoyo/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.samhsa.gov/encuentra-apoyo/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"eldercare.gov","name":"Department of Health and Human Services","redirect":"https://eldercare.acl.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://eldercare.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":502,"redirect":"http://www.eldercare.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"drugabuse.gov","name":"Department of Health and Human Services","redirect":"https://nida.nih.gov/robots.txt","sitemap":"https://nida.nih.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://nida.nih.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://nida.nih.gov/sitemap.xml"}]},{"status":500,"url":"donaciondeorganos.gov","name":"Department of Health and Human Services","redirect":"http://www.donaciondeorganos.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://donaciondeorganos.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://donaciondeorganos.gov/sitemap.xml"},{"time":1743826667324,"status":200,"redirect":"https://donaciondeorganos.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://donaciondeorganos.gov/sitemap.xml","source":"robots.txt"}]},{"status":404,"url":"docline.gov","name":"Department of Health and Human Services","redirect":"https://www.docline.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.docline.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"diabetescommittee.gov","name":"Department of Health and Human Services","redirect":"http://www.diabetescommittee.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":404,"redirect":"https://www.niddk.nih.gov/not-found?404url=www.diabetescommittee.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"dhhs.gov","name":"Department of Health and Human Services","redirect":"https://www.hhs.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.hhs.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"cuidadodesalud.gov","name":"Department of Health and Human Services","redirect":"https://www.cuidadodesalud.gov/es/robots.txt","sitemap":"https://www.cuidadodesalud.gov/es/sitemap-index.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cuidadodesalud.gov/es/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.cuidadodesalud.gov/es/sitemap-index.xml"}]},{"status":200,"url":"covidtests.gov","name":"Department of Health and Human Services","redirect":"https://covidtests.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.covid.gov/tools-and-resources/resources/tests","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":202,"redirect":"https://aspr.hhs.gov/covid-19/test/Pages/default.aspx","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":403,"redirect":"https://covidtests.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"covidtest.gov","name":"Department of Health and Human Services","redirect":"https://covidtest.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.covid.gov/tools-and-resources/resources/tests","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":202,"redirect":"https://aspr.hhs.gov/covid-19/test/Pages/default.aspx","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":403,"redirect":"https://covidtest.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"covid.gov","name":"Department of Health and Human Services","redirect":"https://www.covid.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.covid.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"coronavirus.gov","name":"Department of Health and Human Services","redirect":"https://www.cdc.gov/covid/index.html","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cdc.gov/coronavirus/2019-ncov/index.html","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cdc.gov/covid/index.html","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":302,"redirect":"https://www.coronavirus.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"collegedrinkingprevention.gov","name":"Department of Health and Human Services","redirect":"https://www.collegedrinkingprevention.gov/robots.txt","sitemap":"https://arcr.niaaa.nih.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://collegedrinkingprevention.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://arcr.niaaa.nih.gov/sitemap.xml"}]},{"status":200,"url":"cms.gov","name":"Department of Health and Human Services","redirect":"https://www.cms.gov/robots.txt","sitemap":"https://www.cms.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cms.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"https://www.cms.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"clinicaltrials.gov","name":"Department of Health and Human Services","redirect":"https://clinicaltrials.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://clinicaltrials.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"clinicaltrial.gov","name":"Department of Health and Human Services","redirect":"http://www.clinicaltrial.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"childwelfare.gov","name":"Department of Health and Human Services","redirect":"https://www.childwelfare.gov/robots.txt","sitemap":"https://www.childwelfare.gov/sitemap-index.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"childcare.gov","name":"Department of Health and Human Services","redirect":"https://childcare.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.childcare.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"cerebrosano.gov","name":"Department of Health and Human Services","redirect":"http://www.cerebrosano.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cdcpartners.gov","name":"Department of Health and Human Services","redirect":"https://www.cdcpartners.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cdc.gov","name":"Department of Health and Human Services","redirect":"https://www.cdc.gov/robots.txt","sitemap":"https://www.cdc.gov/wcms-auto-sitemap-index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cdc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.cdc.gov/wcms-auto-sitemap-index.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.cdc.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cancer.gov","name":"Department of Health and Human Services","redirect":"http://www.cancer.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cancer.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.cancer.gov","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":200,"redirect":"https://www.cancer.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1730174316792,"status":500,"redirect":"https://www.cancer.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"brainhealth.gov","name":"Department of Health and Human Services","redirect":"http://www.brainhealth.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"brain.gov","name":"Department of Health and Human Services","redirect":"https://braininitiative.nih.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://braininitiative.nih.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.brain.gov","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":200,"redirect":"https://braininitiative.nih.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1730174316792,"status":500,"redirect":"http://www.brain.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":200,"redirect":"https://braininitiative.nih.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":500,"redirect":"http://www.brain.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"birthcontrol.gov","name":"Department of Health and Human Services","redirect":"https://birthcontrol.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://reproductivehealthservices.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.birthcontrol.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"bioethics.gov","name":"Department of Health and Human Services","redirect":"http://www.bioethics.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"betobaccofree.gov","name":"Department of Health and Human Services","redirect":"https://betobaccofree.hhs.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://betobaccofree.hhs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://betobaccofree.hhs.gov/sitemap.xml"},{"time":1728949876846,"status":500,"redirect":"http://www.betobaccofree.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.betobaccofree.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739509031548,"status":200,"redirect":"https://betobaccofree.hhs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://betobaccofree.hhs.gov/sitemap.xml","source":"robots.txt"}]},{"status":500,"url":"bam.gov","name":"Department of Health and Human Services","redirect":"http://www.bam.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"aspr.gov","name":"Department of Health and Human Services","redirect":"https://www.aspr.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://aspr.hhs.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://aspr.hhs.gov:443/sitemap.xml"},{"time":1728949876846,"status":202,"redirect":"https://aspr.hhs.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"arpah.gov","name":"Department of Health and Human Services","redirect":"https://arpa-h.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"http://www.arpah.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"arpa-h.gov","name":"Department of Health and Human Services","redirect":"https://arpa-h.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://arpa-h.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"aoa.gov","name":"Department of Health and Human Services","redirect":"http://www.aoa.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":405,"url":"alzheimers.gov","name":"Department of Health and Human Services","redirect":"https://www.alzheimers.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1737430190371,"status":403,"redirect":"https://www.alzheimers.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"aids.gov","name":"Department of Health and Human Services","redirect":"https://www.hiv.gov/robots.txt","sitemap":"https://www.hiv.gov/sitemap-index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.hiv.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.hiv.gov/sitemap-index.xml"}]},{"status":403,"url":"ahrq.gov","name":"Department of Health and Human Services","redirect":"https://www.ahrq.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ahrq.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":403,"url":"ahcpr.gov","name":"Department of Health and Human Services","redirect":"https://www.ahcpr.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ahrq.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":502,"url":"agingstats.gov","name":"Department of Health and Human Services","redirect":"http://www.agingstats.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"aging.gov","name":"Department of Health and Human Services","redirect":"https://www.hhs.gov/aging/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"afterschool.gov","name":"Department of Health and Human Services","redirect":"http://www.afterschool.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"acl.gov","name":"Department of Health and Human Services","redirect":"https://acl.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://acl.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://acl.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"acf.gov","name":"Department of Health and Human Services","redirect":"https://acf.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.acf.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"988.gov","name":"Department of Health and Human Services","redirect":"https://988.gov/robots.txt","sitemap":"https://988lifeline.org/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://988lifeline.org/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"ymp.gov","name":"Department of Energy","redirect":"http://www.ymp.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"wapa.gov","name":"Department of Energy","redirect":"https://www.wapa.gov/robots.txt","sitemap":"https://www.wapa.gov/sitemap.xml","history":[{"time":1728949876846,"status":500,"redirect":"http://www.wapa.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"unrpnet.gov","name":"Department of Energy","redirect":"http://www.unrpnet.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"unnpp.gov","name":"Department of Energy","redirect":"http://www.unnpp.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"swpa.gov","name":"Department of Energy","redirect":"http://www.swpa.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"srs.gov","name":"Department of Energy","redirect":"https://www.srs.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"srnl.gov","name":"Department of Energy","redirect":"https://www.srnl.gov/robots.txt","sitemap":"https://www.srnl.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.srnl.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.srnl.gov/sitemap_index.xml"}]},{"status":500,"url":"solardecathlon.gov","name":"Department of Energy","redirect":"https://www.solardecathlon.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"https://www.solardecathlon.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"sns.gov","name":"Department of Energy","redirect":"https://sns.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://sns.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"smartgrid.gov","name":"Department of Energy","redirect":"https://www.energy.gov/grid-talk/grid-talk?nrg_redirect=463876","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://smartgrid.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"https://smartgrid.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.smartgrid.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":403,"redirect":"https://www.energy.gov/electricity-insights","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"science.gov","name":"Department of Energy","redirect":"https://science.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.science.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"scidac.gov","name":"Department of Energy","redirect":"https://scidac.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://scidac.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"sandia.gov","name":"Department of Energy","redirect":"https://www.sandia.gov/robots.txt","sitemap":"https://www.sandia.gov/wp-sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.sandia.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.sandia.gov/wp-sitemap.xml"}]},{"status":500,"url":"safgrandchallenge.gov","name":"Department of Energy","redirect":"http://www.safgrandchallenge.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"rl.gov","name":"Department of Energy","redirect":"http://www.rl.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"rideelectric.gov","name":"Department of Energy","redirect":"https://driveelectric.gov/ride","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://driveelectric.gov/ride","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://driveelectric.gov/ride","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://rideelectric.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"pr100.gov","name":"Department of Energy","redirect":"https://pr100.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://pr100.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"pppo.gov","name":"Department of Energy","redirect":"http://www.pppo.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"pppl.gov","name":"Department of Energy","redirect":"https://www.pppl.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"pnnl.gov","name":"Department of Energy","redirect":"https://www.pnnl.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pnnl.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"pnl.gov","name":"Department of Energy","redirect":"https://www.pnnl.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pnnl.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"pcast.gov","name":"Department of Energy","redirect":"http://www.pcast.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"osti.gov","name":"Department of Energy","redirect":"https://www.osti.gov/robots.txt","sitemap":"https://www.osti.gov/sitemap_ostigov/xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.osti.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.osti.gov/sitemap_ostigov.xml"},{"time":1734148194754,"status":200,"redirect":"https://www.osti.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.osti.gov/sitemap_ostigov.xml","source":"robots.txt"}]},{"status":200,"url":"ornl.gov","name":"Department of Energy","redirect":"https://www.ornl.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ornl.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"orau.gov","name":"Department of Energy","redirect":"https://orau.gov/robots.txt","sitemap":"http://orise.orau.gov/nsf-msgi/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://orau.gov/robots.txt","valid":true,"allowed":true,"sitemap":"http://orise.orau.gov/nsf-msgi/sitemap.xml"}]},{"status":500,"url":"nuclear.gov","name":"Department of Energy","redirect":"https://www.nuclear.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ntrc.gov","name":"Department of Energy","redirect":"http://www.ntrc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nrelhub.gov","name":"Department of Energy","redirect":"http://www.nrelhub.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"nrel.gov","name":"Department of Energy","redirect":"https://www.nrel.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.nrel.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":404,"redirect":"https://www.nrel.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1737430190371,"status":404,"redirect":"https://www.nrel.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nnss.gov","name":"Department of Energy","redirect":"https://nnss.gov/robots.txt","sitemap":"https://nnss.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://nnss.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://nnss.gov/sitemap_index.xml"}]},{"valid":true,"source":"meta","allowed":true,"status":500,"url":"neup.gov","name":"Department of Energy","redirect":"https://neup.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.neup.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"nersc.gov","name":"Department of Energy","redirect":"https://www.nersc.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.nersc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"ncrc.gov","name":"Department of Energy","redirect":"https://www.ncrc.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"nccs.gov","name":"Department of Energy","redirect":"http://www.nccs.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"llnl.gov","name":"Department of Energy","redirect":"https://www.llnl.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.llnl.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"lbl.gov","name":"Department of Energy","redirect":"https://www.lbl.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.lbl.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"lanl.gov","name":"Department of Energy","redirect":"https://www.lanl.gov/robots.txt","sitemap":"https://www.lanl.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.lanl.gov","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://lanl.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"isotopes.gov","name":"Department of Energy","redirect":"https://isotopes.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://isotopes.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"isotope.gov","name":"Department of Energy","redirect":"http://www.isotope.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"inl.gov","name":"Department of Energy","redirect":"https://inl.gov/robots.txt","sitemap":"https://inl.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://inl.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://inl.gov/sitemap_index.xml"},{"time":1728949876846,"status":403,"redirect":"http://www.inl.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"inel.gov","name":"Department of Energy","redirect":"http://www.inel.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"hydrogen.gov","name":"Department of Energy","redirect":"https://www.hydrogen.energy.gov/interagency","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.hydrogen.energy.gov/interagency","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://www.hydrogen.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"homeenergyscore.gov","name":"Department of Energy","redirect":"https://betterbuildingssolutioncenter.energy.gov/home-energy-score","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"hanford.gov","name":"Department of Energy","redirect":"http://www.hanford.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://hanford.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"fueleconomy.gov","name":"Department of Energy","redirect":"https://fueleconomy.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://fueleconomy.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"fnal.gov","name":"Department of Energy","redirect":"https://fnal.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://fnal.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"energysavers.gov","name":"Department of Energy","redirect":"https://www.energy.gov/energysaver/energy-saver","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":403,"redirect":"https://www.energy.gov/energysaver/energy-saver","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":500,"redirect":"https://www.energysavers.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"energysaver.gov","name":"Department of Energy","redirect":"https://www.energy.gov/energysaver/energy-saver/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1739509031548,"status":500,"redirect":"https://www.energysaver.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"energycommunities.gov","name":"Department of Energy","redirect":"https://energycommunities.gov/robots.txt","sitemap":"https://energycommunities.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"http://energycommunities.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://energycommunities.gov/sitemap_index.xml"}]},{"status":200,"url":"energycodes.gov","name":"Department of Energy","redirect":"https://energycodes.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.energycodes.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"energy.gov","name":"Department of Energy","redirect":"https://www.energy.gov/robots.txt","sitemap":null,"history":[{"time":1739509031548,"status":403,"redirect":"https://www.energy.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"eia.gov","name":"Department of Energy","redirect":"https://www.eia.gov/robots.txt","sitemap":"https://www.eia.gov/sitemap_eia04_2021.xml","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"driveelectric.gov","name":"Department of Energy","redirect":"https://driveelectric.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://driveelectric.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"doe.gov","name":"Department of Energy","redirect":"https://www.energy.gov/robots.txt","sitemap":null,"history":[{"time":1728949876846,"status":403,"redirect":"https://www.energy.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":302,"redirect":"https://www.doe.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":403,"redirect":"https://www.energy.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"citap.gov","name":"Department of Energy","redirect":"https://citap.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://citap.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"cendi.gov","name":"Department of Energy","redirect":"https://cendi.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://cendi.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"cebaf.gov","name":"Department of Energy","redirect":"http://www.cebaf.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"casl.gov","name":"Department of Energy","redirect":"http://www.casl.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"buildingamerica.gov","name":"Department of Energy","redirect":"http://www.buildingamerica.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"bpa.gov","name":"Department of Energy","redirect":"https://www.bpa.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.bpa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.bpa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1734148194754,"status":500,"redirect":"https://www.bpa.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":200,"redirect":"https://www.bpa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"bnl.gov","name":"Department of Energy","redirect":"https://www.bnl.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.bnl.gov/world/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"biomassboard.gov","name":"Department of Energy","redirect":"https://biomassboard.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":404,"redirect":"http://biomassboard.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"arm.gov","name":"Department of Energy","redirect":"https://arm.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://arm.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1740965653672,"status":200,"redirect":"https://arm.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1742146243896,"status":500,"redirect":"https://www.arm.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"anl.gov","name":"Department of Energy","redirect":"https://www.anl.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ameslab.gov","name":"Department of Energy","redirect":"https://www.ameslab.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"studentloans.gov","name":"Department of Education","redirect":"https://studentaid.gov/robots.txt","sitemap":"https://studentaid.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.studentloans.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"studentaid.gov","name":"Department of Education","redirect":"https://studentaid.gov/robots.txt","sitemap":"https://studentaid.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://studentaid.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://studentaid.gov/sitemap.xml"}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"nationsreportcard.gov","name":"Department of Education","redirect":"https://www.nationsreportcard.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.nationsreportcard.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":404,"redirect":"https://www.nationsreportcard.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"nagb.gov","name":"Department of Education","redirect":"https://www.nagb.gov/robots.txt","sitemap":"https://www.nagb.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nagb.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.nagb.gov/sitemap.xml"}]},{"status":200,"url":"g5.gov","name":"Department of Education","redirect":"https://g5.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://g5.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.g5.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fafsa.gov","name":"Department of Education","redirect":"https://studentaid.gov/h/apply-for-aid/fafsa","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://studentaid.gov/h/apply-for-aid/fafsa","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://studentaid.gov/h/apply-for-aid/fafsa","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.fafsa.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"ed.gov","name":"Department of Education","redirect":"https://www.ed.gov/sites/ed/files/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ed.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.ed.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":404,"redirect":"https://www.ed.gov/sites/ed/files/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1739581966243,"status":500,"redirect":"http://www.ed.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://www.ed.gov/sites/ed/files/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":500,"redirect":"http://www.ed.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"collegenavigator.gov","name":"Department of Education","redirect":"https://nces.ed.gov/collegenavigator/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://nces.ed.gov/collegenavigator/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://nces.ed.gov/collegenavigator/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://www.collegenavigator.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1739509031548,"status":200,"redirect":"https://nces.ed.gov/collegenavigator/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1742146243896,"status":500,"redirect":"http://collegenavigator.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":404,"url":"childstats.gov","name":"Department of Education","redirect":"https://www.childstats.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.childstats.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"https://www.childstats.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"budgetlob.gov","name":"Department of Education","redirect":"http://www.budgetlob.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"usandc.gov","name":"Department of Defense","redirect":"http://www.usandc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"tswg.gov","name":"Department of Defense","redirect":"https://tswg.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://tswg.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.tswg.gov","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":404,"redirect":"https://tswg.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":525,"redirect":"https://www.tswg.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":400,"url":"tak.gov","name":"Department of Defense","redirect":"http://www.tak.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":421,"redirect":"https://tak.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.tak.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"http://www.tak.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":421,"redirect":"https://tak.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":400,"redirect":"http://www.tak.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":421,"redirect":"https://tak.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"oneaftac.gov","name":"Department of Defense","redirect":"http://www.oneaftac.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"oldcc.gov","name":"Department of Defense","redirect":"http://www.oldcc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"oea.gov","name":"Department of Defense","redirect":"http://www.oea.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nsep.gov","name":"Department of Defense","redirect":"http://www.nsep.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nsa.gov","name":"Department of Defense","redirect":"https://www.nsa.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nsa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://nsa.gov/DesktopModules/SiteData/SiteMap.ashx"},{"time":1728949876846,"status":200,"redirect":"https://www.nsa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://nsa.gov/DesktopModules/SiteData/SiteMap.ashx","source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nro.gov","name":"Department of Defense","redirect":"https://www.nro.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.nro.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://nro.gov/DesktopModules/SiteData/SiteMap.ashx","source":"robots.txt"}]},{"status":200,"url":"nrd.gov","name":"Department of Defense","redirect":"https://nrd.gov/robots.txt","sitemap":"https://nrd.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"http://nrd.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":404,"redirect":"https://nrd.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"meta","allowed":true,"status":200,"url":"nbib.gov","name":"Department of Defense","redirect":"https://www.dcsa.mil/","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dcsa.mil/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.dcsa.mil/","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":301,"redirect":"https://nbib.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":200,"url":"nationalresourcedirectory.gov","name":"Department of Defense","redirect":"http://nrd.gov/robots.txt","sitemap":"https://nrd.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"http://nationalresourcedirectory.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":404,"redirect":"https://nationalresourcedirectory.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"mypay.gov","name":"Department of Defense","redirect":"http://www.mypay.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"mtmc.gov","name":"Department of Defense","redirect":"http://www.mtmc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"lps.gov","name":"Department of Defense","redirect":"https://www.lps.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.lps.gov","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":404,"redirect":"http://www.lps.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1730174316792,"status":500,"redirect":"http://www.lps.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":404,"redirect":"http://www.lps.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"}]},{"status":500,"url":"jccs.gov","name":"Department of Defense","redirect":"http://www.jccs.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"iwtsd.gov","name":"Department of Defense","redirect":"https://www.iwtsd.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://iwtsd.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"itc.gov","name":"Department of Defense","redirect":"http://www.itc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ioss.gov","name":"Department of Defense","redirect":"https://www.ioss.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"intelligencecareers.gov","name":"Department of Defense","redirect":"https://www.intelligencecareers.gov/robots.txt","sitemap":"https://www.usajobs.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"iad.gov","name":"Department of Defense","redirect":"https://www.iad.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"hive.gov","name":"Department of Defense","redirect":"https://www.hive.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"x-robots-tag","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"fvap.gov","name":"Department of Defense","redirect":"https://www.fvap.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.fvap.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fehrm.gov","name":"Department of Defense","redirect":"https://www.fehrm.gov/robots.txt","sitemap":"https://www.fehrm.gov/sitemap-index.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"defense.gov","name":"Department of Defense","redirect":"https://www.defense.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.defense.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://defense.gov/DesktopModules/SiteData/SiteMap.ashx"},{"time":1728949876846,"status":200,"redirect":"https://www.defense.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://defense.gov/DesktopModules/SiteData/SiteMap.ashx","source":"robots.txt"}]},{"status":500,"url":"dc3on.gov","name":"Department of Defense","redirect":"http://www.dc3on.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"cttso.gov","name":"Department of Defense","redirect":"https://cttso.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://cttso.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ctoc.gov","name":"Department of Defense","redirect":"https://ctoc.gov/robots.txt","sitemap":"https://ctoc.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"cnss.gov","name":"Department of Defense","redirect":"https://www.cnss.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"cmts.gov","name":"Department of Defense","redirect":"https://www.cmts.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cmts.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cmts.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://cmts.gov/DesktopModules/SiteData/SiteMap.ashx","source":"robots.txt"}]},{"status":404,"url":"capnhq.gov","name":"Department of Defense","redirect":"https://capnhq.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://capnhq.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"cap.gov","name":"Department of Defense","redirect":"http://www.cap.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"businessdefense.gov","name":"Department of Defense","redirect":"http://www.businessdefense.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"altusandc.gov","name":"Department of Defense","redirect":"http://www.altusandc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"aftac.gov","name":"Department of Defense","redirect":"http://www.aftac.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"adlnet.gov","name":"Department of Defense","redirect":"https://adlnet.gov/robots.txt","sitemap":"https://adlnet.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://adlnet.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://adlnet.gov/sitemap.xml"}]},{"status":200,"url":"xd.gov","name":"Department of Commerce","redirect":"https://www.xd.gov/robots.txt","sitemap":"https://xd.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.xd.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"wwtg.gov","name":"Department of Commerce","redirect":"https://www.wwtg.gov/robots.txt","sitemap":"https://www.wwtg.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.wwtg.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.wwtg.gov/sitemap.xml"}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"weather.gov","name":"Department of Commerce","redirect":"https://www.weather.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.weather.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"uspto.gov","name":"Department of Commerce","redirect":"https://www.uspto.gov/robots.txt","sitemap":"https://www.uspto.gov/xfer-search/sitemap.xml","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"usicecenter.gov","name":"Department of Commerce","redirect":"https://usicecenter.gov/robots.txt","sitemap":"https://usicecenter.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://usicecenter.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://usicecenter.gov/sitemap.xml"}]},{"status":200,"url":"tsunami.gov","name":"Department of Commerce","redirect":"https://tsunami.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://tsunami.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"trade.gov","name":"Department of Commerce","redirect":"https://www.trade.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://trade.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"time.gov","name":"Department of Commerce","redirect":"https://time.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.time.gov","valid":false,"allowed":false,"sitemap":null},{"time":1742146243896,"status":404,"redirect":"https://time.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":500,"redirect":"https://time.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"techhubs.gov","name":"Department of Commerce","redirect":"https://www.eda.gov/funding/programs/regional-technology-and-innovation-hubs","sitemap":null,"valid":true,"allowed":true,"source":"x-robots-tag","history":[{"time":1726617445612,"status":200,"redirect":"https://www.eda.gov/funding/programs/regional-technology-and-innovation-hubs","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.eda.gov/funding/programs/regional-technology-and-innovation-hubs","valid":true,"allowed":true,"sitemap":null,"source":"x-robots-tag"},{"time":1730174316792,"status":301,"redirect":"https://www.techhubs.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"x-robots-tag"}]},{"status":404,"url":"tasefiling.gov","name":"Department of Commerce","redirect":"https://tasefiling.powerappsportals.us/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://tasefiling.powerappsportals.us/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"sworm.gov","name":"Department of Commerce","redirect":"https://www.sworm.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.sworm.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"stopfakes.gov","name":"Department of Commerce","redirect":"http://www.stopfakes.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.stopfakes.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.stopfakes.gov/sitemap"},{"time":1728949876846,"status":200,"redirect":"https://www.stopfakes.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.stopfakes.gov/sitemap","source":"robots.txt"}]},{"status":200,"url":"standards.gov","name":"Department of Commerce","redirect":"https://www.nist.gov/standardsgov","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nist.gov/standardsgov","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.nist.gov/standardsgov","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":307,"redirect":"https://standards.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1742146243896,"status":200,"redirect":"https://www.nist.gov/standardsgov","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1743826667324,"status":500,"redirect":"http://www.standards.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"spectrum.gov","name":"Department of Commerce","redirect":"https://www.ntia.gov/other-publication/2017/federal-government-spectrum-compendium","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ntia.gov/other-publication/2017/federal-government-spectrum-compendium","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.ntia.gov/other-publication/2017/federal-government-spectrum-compendium","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.spectrum.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"spd15revision.gov","name":"Department of Commerce","redirect":"https://spd15revision.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1742146243896,"status":500,"redirect":"http://www.spd15revision.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"spaceweather.gov","name":"Department of Commerce","redirect":"https://www.spaceweather.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.spaceweather.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"semiconductors.gov","name":"Department of Commerce","redirect":"https://www.nist.gov/chips","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nist.gov/chips","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.semiconductors.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":500,"redirect":"http://www.semiconductors.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":500,"redirect":"http://www.semiconductors.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"selectusa.gov","name":"Department of Commerce","redirect":"https://www.trade.gov/selectusa","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.trade.gov/selectusa-home","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.trade.gov/selectusa-home","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1729037740707,"status":403,"redirect":"https://www.trade.gov/selectusa","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":302,"redirect":"https://www.selectusa.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":403,"redirect":"https://www.trade.gov/selectusa","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"sdr.gov","name":"Department of Commerce","redirect":"https://www.sdr.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.sdr.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"pscr.gov","name":"Department of Commerce","redirect":"https://www.nist.gov/ctl/pscr","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nist.gov/ctl/pscr","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.nist.gov/ctl/pscr","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.pscr.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"privacyshield.gov","name":"Department of Commerce","redirect":"http://www.privacyshield.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.privacyshield.gov/ps/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.privacyshield.gov/sitemap"},{"time":1728949876846,"status":200,"redirect":"https://www.privacyshield.gov/ps/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.privacyshield.gov/sitemap","source":"robots.txt"}]},{"status":200,"url":"papahanaumokuakea.gov","name":"Department of Commerce","redirect":"https://www.papahanaumokuakea.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.papahanaumokuakea.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"ofcm.gov","name":"Department of Commerce","redirect":"https://www.www.ofcm.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"nwirp.gov","name":"Department of Commerce","redirect":"http://www.nwirp.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"ntis.gov","name":"Department of Commerce","redirect":"https://www.ntis.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1728949876846,"status":500,"redirect":"http://www.ntis.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":404,"redirect":"https://www.ntis.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":403,"redirect":"http://www.ntis.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ntia.gov","name":"Department of Commerce","redirect":"https://www.ntia.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"http://ntia.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"noaa.gov","name":"Department of Commerce","redirect":"https://www.noaa.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.noaa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"nist.gov","name":"Department of Commerce","redirect":"https://www.nist.gov/robots.txt","sitemap":"https://www.nist.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nist.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.nist.gov/sitemap.xml"}]},{"status":200,"url":"nehrp.gov","name":"Department of Commerce","redirect":"https://nehrp.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://nehrp.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1729037740707,"status":500,"redirect":"https://www.nehrp.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.nehrp.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1737430190371,"status":500,"redirect":"https://www.nehrp.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":400,"url":"my2020census.gov","name":"Department of Commerce","redirect":"http://www.my2020census.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"mgi.gov","name":"Department of Commerce","redirect":"https://www.mgi.gov/robots.txt","sitemap":"https://www.nist.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.mgi.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1729037740707,"status":200,"redirect":"https://www.mgi.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1730174316792,"status":500,"redirect":"http://www.mgi.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1734148194754,"status":200,"redirect":"https://www.mgi.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1737430190371,"status":500,"redirect":"http://www.mgi.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"mbda.gov","name":"Department of Commerce","redirect":"https://www.mbda.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.mbda.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"marinecadastre.gov","name":"Department of Commerce","redirect":"https://hub.marinecadastre.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://marinecadastre.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.marinecadastre.gov","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://www.marinecadastre.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"manufacturing.gov","name":"Department of Commerce","redirect":"https://www.manufacturing.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.manufacturing.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"luca-appeals.gov","name":"Department of Commerce","redirect":"http://www.luca-appeals.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"internetforall.gov","name":"Department of Commerce","redirect":"https://www.internetforall.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.internetforall.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"internet4all.gov","name":"Department of Commerce","redirect":"https://www.internetforall.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.internetforall.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"iedison.gov","name":"Department of Commerce","redirect":"https://www.nist.gov/iedison","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.nist.gov/iedison/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://www.nist.gov/iedison/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"icts.gov","name":"Department of Commerce","redirect":"http://www.icts.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"icams-portal.gov","name":"Department of Commerce","redirect":"https://www.icams-portal.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":403,"redirect":"https://www.icams-portal.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"hurricanes.gov","name":"Department of Commerce","redirect":"https://www.nhc.noaa.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":503,"redirect":"https://www.nhc.noaa.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":503,"redirect":"https://www.nhc.noaa.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1734148194754,"status":503,"redirect":"https://www.nhc.noaa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"meta"},{"time":1740965653672,"status":200,"redirect":"https://www.nhc.noaa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1743826667324,"status":500,"redirect":"https://www.flash.org/hurricanestrong","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"heat.gov","name":"Department of Commerce","redirect":"https://www.heat.gov/robots.txt","sitemap":"https://www.heat.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"gps.gov","name":"Department of Commerce","redirect":"https://www.gps.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.gps.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"goes-r.gov","name":"Department of Commerce","redirect":"https://www.goes-r.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.goes-r.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"fishwatch.gov","name":"Department of Commerce","redirect":"https://www.fisheries.noaa.gov/topic/sustainable-seafood","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.fisheries.noaa.gov/topic/sustainable-seafood","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.fisheries.noaa.gov/topic/sustainable-seafood","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.fishwatch.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"firstnet.gov","name":"Department of Commerce","redirect":"https://firstnet.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":403,"redirect":"http://firstnet.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"export.gov","name":"Department of Commerce","redirect":"http://www.export.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.trade.gov/export-solutions","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.trade.gov/export-solutions","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"edison.gov","name":"Department of Commerce","redirect":"https://www.nist.gov/iedison","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.nist.gov/iedison/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":500,"redirect":"https://www.edison.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://www.nist.gov/iedison/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"eda.gov","name":"Department of Commerce","redirect":"https://www.eda.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.eda.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":401,"url":"drought.gov","name":"Department of Commerce","redirect":"https://www.drought.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.drought.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://www.drought.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"doc.gov","name":"Department of Commerce","redirect":"https://www.commerce.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.commerce.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"https://www.doc.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"dnsops.gov","name":"Department of Commerce","redirect":"http://www.dnsops.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"dataprivacyframework.gov","name":"Department of Commerce","redirect":"https://www.dataprivacyframework.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.dataprivacyframework.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.dataprivacyframework.gov","valid":false,"allowed":false,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"https://www.dataprivacyframework.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":404,"url":"cwc.gov","name":"Department of Commerce","redirect":"https://www.cwc.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.cwc.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.cwc.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"commerce.gov","name":"Department of Commerce","redirect":"https://www.commerce.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.commerce.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"http://www.commerce.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"climate.gov","name":"Department of Commerce","redirect":"https://www.climate.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.climate.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"chips.gov","name":"Department of Commerce","redirect":"https://www.nist.gov/chips","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.nist.gov/chips/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":500,"redirect":"http://www.chips.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":500,"redirect":"https://www.chips.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1737430190371,"status":500,"redirect":"http://www.chips.gov/","valid":false,"allowed":false,"sitemap":null},{"time":1740965653672,"status":404,"redirect":"https://www.nist.gov/chips/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"census.gov","name":"Department of Commerce","redirect":"https://www.census.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.census.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"buyusa.gov","name":"Department of Commerce","redirect":"http://www.buyusa.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"bldrdoc.gov","name":"Department of Commerce","redirect":"http://www.bldrdoc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"bis.gov","name":"Department of Commerce","redirect":"https://www.bis.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.bis.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://www.bis.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"bea.gov","name":"Department of Commerce","redirect":"https://www.bea.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.bea.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"aviationweather.gov","name":"Department of Commerce","redirect":"https://aviationweather.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://aviationweather.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"ap.gov","name":"Department of Commerce","redirect":"https://www.bis.doc.gov/index.php/other-areas/additional-protocol-ap","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.bis.doc.gov/index.php/other-areas/additional-protocol-ap","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.bis.doc.gov/index.php/other-areas/additional-protocol-ap","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":301,"redirect":"https://www.ap.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"2020census.gov","name":"Department of Commerce","redirect":"http://www.2020census.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":503,"redirect":"https://2020census.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"denali.gov","name":"Denali Commission","redirect":"https://denali.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"http://denali.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"dra.gov","name":"Delta Regional Authority","redirect":"https://dra.gov/robots.txt","sitemap":"https://dra.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://dra.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://dra.gov/sitemap_index.xml"}]},{"status":200,"url":"dnfsb.gov","name":"Defense Nuclear Facilities Safety Board","redirect":"http://www.dnfsb.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"psa.gov","name":"Court Services and Offender Supervision","redirect":"https://psa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.psa.gov","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"pretrialservices.gov","name":"Court Services and Offender Supervision","redirect":"https://pretrialservices.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1728949876846,"status":500,"redirect":"http://www.pretrialservices.gov","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"csosa.gov","name":"Court Services and Offender Supervision","redirect":"https://www.csosa.gov/robots.txt","sitemap":"https://www.csosa.gov/wp-sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.csosa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.csosa.gov/wp-sitemap.xml"}]},{"status":200,"url":"pandemicoversight.gov","name":"Council of Inspectors General on Integrity and Efficiency","redirect":"https://pandemicoversight.gov/robots.txt","sitemap":"https://pandemicoversight.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pandemicoversight.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://pandemicoversight.gov/sitemap.xml"}]},{"status":200,"url":"oversight.gov","name":"Council of Inspectors General on Integrity and Efficiency","redirect":"https://www.oversight.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.oversight.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"ignet.gov","name":"Council of Inspectors General on Integrity and Efficiency","redirect":"https://www.ignet.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ignet.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"cigie.gov","name":"Council of Inspectors General on Integrity and Efficiency","redirect":"http://www.cigie.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"volunteeringinamerica.gov","name":"Corporation for National and Community Service","redirect":"https://volunteeringinamerica.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"vistacampus.gov","name":"Corporation for National and Community Service","redirect":"https://vistacampus.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"serve.gov","name":"Corporation for National and Community Service","redirect":"https://serve.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"presidentialserviceawards.gov","name":"Corporation for National and Community Service","redirect":"https://d302laedhb14lz.cloudfront.net/f75e0a14-4aba-47e2-a5d8-18d1e84b491b/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://d302laedhb14lz.cloudfront.net/2f1df44d-a1a0-4e2d-a6f5-434f1cea4c0a/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"nationalservice.gov","name":"Corporation for National and Community Service","redirect":"https://nationalservice.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"mlkday.gov","name":"Corporation for National and Community Service","redirect":"https://mlkday.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"mentor.gov","name":"Corporation for National and Community Service","redirect":"https://mentor.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"joinamericorps.gov","name":"Corporation for National and Community Service","redirect":"https://joinamericorps.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cns.gov","name":"Corporation for National and Community Service","redirect":"http://www.cns.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":404,"url":"cncsoig.gov","name":"Corporation for National and Community Service","redirect":"http://www.cncsoig.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"cncs.gov","name":"Corporation for National and Community Service","redirect":"http://www.cncs.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ccc.gov","name":"Corporation for National and Community Service","redirect":"http://www.ccc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"americorpsoig.gov","name":"Corporation for National and Community Service","redirect":"https://www.americorpsoig.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.americorpsoig.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"americorps.gov","name":"Corporation for National and Community Service","redirect":"https://americorps.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://americorps.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"americanclimatecorps.gov","name":"Corporation for National and Community Service","redirect":"http://www.americanclimatecorps.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.acc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.acc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1740965653672,"status":404,"redirect":"http://www.americanclimatecorps.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"acc.gov","name":"Corporation for National and Community Service","redirect":"http://www.acc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.acc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.acc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1740965653672,"status":404,"redirect":"http://www.acc.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"seguridadconsumidor.gov","name":"Consumer Product Safety Commission","redirect":"https://www.cpsc.gov/es","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cpsc.gov/es/SeguridadConsumidor","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cpsc.gov/es/SeguridadConsumidor","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://seguridadconsumidor.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"saferproducts.gov","name":"Consumer Product Safety Commission","redirect":"https://www.saferproducts.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.saferproducts.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://www.saferproducts.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"saferproduct.gov","name":"Consumer Product Safety Commission","redirect":"https://www.saferproducts.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.saferproducts.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":404,"redirect":"https://www.saferproducts.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"recalls.gov","name":"Consumer Product Safety Commission","redirect":"https://www.recalls.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.recalls.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"poolsafety.gov","name":"Consumer Product Safety Commission","redirect":"https://www.poolsafely.gov/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.poolsafely.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.poolsafely.gov/","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":404,"redirect":"https://www.poolsafely.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":404,"url":"poolsafely.gov","name":"Consumer Product Safety Commission","redirect":"https://www.poolsafely.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.poolsafely.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"drywallresponse.gov","name":"Consumer Product Safety Commission","redirect":"https://www.cpsc.gov/Safety-Education/Safety-Education-Centers/Drywall-Information-Center","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cpsc.gov/Safety-Education/Safety-Education-Centers/Drywall-Information-Center","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cpsc.gov/Safety-Education/Safety-Education-Centers/Drywall-Information-Center","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://drywallresponse.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"cpsc.gov","name":"Consumer Product Safety Commission","redirect":"https://www.cpsc.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.cpsc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"atvsafety.gov","name":"Consumer Product Safety Commission","redirect":"https://www.cpsc.gov/Safety-Education/Safety-Education-Centers/ATV-Safety-Information-Center","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cpsc.gov/Safety-Education/Safety-Education-Centers/ATV-Safety-Information-Center","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cpsc.gov/Safety-Education/Safety-Education-Centers/ATV-Safety-Information-Center","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1730174316792,"status":302,"redirect":"https://atvsafety.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"valid":true,"source":"meta","allowed":true,"status":404,"url":"anchorit.gov","name":"Consumer Product Safety Commission","redirect":"https://www.anchorit.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.anchorit.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"mimm.gov","name":"Consumer Financial Protection Bureau","redirect":"https://www.consumerfinance.gov/consumer-tools/military-financial-lifecycle/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.mimm.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1737430190371,"status":200,"redirect":"https://www.mimm.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"consumerprotectionbureau.gov","name":"Consumer Financial Protection Bureau","redirect":"https://www.consumerfinance.gov/robots.txt","sitemap":"https://www.consumerfinance.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.consumerfinance.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":404,"redirect":"https://www.consumerfinance.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"consumerprotection.gov","name":"Consumer Financial Protection Bureau","redirect":"https://www.consumerfinance.gov/robots.txt","sitemap":"https://www.consumerfinance.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.consumerfinance.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":404,"redirect":"https://www.consumerfinance.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"consumerfinancialprotectionbureau.gov","name":"Consumer Financial Protection Bureau","redirect":"https://www.consumerfinance.gov/robots.txt","sitemap":"https://www.consumerfinance.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.consumerfinance.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":404,"redirect":"https://www.consumerfinance.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"consumerfinancialbureau.gov","name":"Consumer Financial Protection Bureau","redirect":"https://www.consumerfinance.gov/robots.txt","sitemap":"https://www.consumerfinance.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.consumerfinance.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":404,"redirect":"https://www.consumerfinance.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"consumerfinancial.gov","name":"Consumer Financial Protection Bureau","redirect":"https://www.consumerfinance.gov/robots.txt","sitemap":"https://www.consumerfinance.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.consumerfinance.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":404,"redirect":"https://www.consumerfinance.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"consumerfinance.gov","name":"Consumer Financial Protection Bureau","redirect":"https://www.consumerfinance.gov/robots.txt","sitemap":"https://www.consumerfinance.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.consumerfinance.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":404,"redirect":"https://www.consumerfinance.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"consumerbureau.gov","name":"Consumer Financial Protection Bureau","redirect":"https://www.consumerfinance.gov/robots.txt","sitemap":"https://www.consumerfinance.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.consumerfinance.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":404,"redirect":"https://www.consumerfinance.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"cfpb.gov","name":"Consumer Financial Protection Bureau","redirect":"https://www.consumerfinance.gov/robots.txt","sitemap":"https://www.consumerfinance.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.consumerfinance.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":404,"redirect":"https://www.consumerfinance.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"cfpa.gov","name":"Consumer Financial Protection Bureau","redirect":"https://www.consumerfinance.gov/robots.txt","sitemap":"https://www.consumerfinance.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.consumerfinance.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":404,"redirect":"https://www.consumerfinance.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"bcfp.gov","name":"Consumer Financial Protection Bureau","redirect":"https://www.consumerfinance.gov/robots.txt","sitemap":"https://www.consumerfinance.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":500,"redirect":null,"valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1739509031548,"status":200,"redirect":"https://www.consumerfinance.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.consumerfinance.gov/sitemap.xml","source":"robots.txt"},{"time":1742146243896,"status":404,"redirect":"https://www.consumerfinance.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"whistleblower.gov","name":"Commodity Futures Trading Commission","redirect":"https://www.whistleblower.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.whistleblower.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":403,"url":"smartcheck.gov","name":"Commodity Futures Trading Commission","redirect":"https://www.cftc.gov/LearnAndProtect","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cftc.gov/LearnAndProtect","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.cftc.gov/LearnAndProtect","valid":false,"allowed":false,"sitemap":null,"source":false},{"time":1729037740707,"status":301,"redirect":"https://www.smartcheck.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":500,"url":"devcftc.gov","name":"Commodity Futures Trading Commission","redirect":"http://www.devcftc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":403,"url":"cftc.gov","name":"Commodity Futures Trading Commission","redirect":"https://www.cftc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.cftc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"http://www.cftc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":404,"url":"coldcaserecords.gov","name":"Civil Rights Cold Case Records Review Board","redirect":"https://www.coldcaserecords.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"x-robots-tag","history":[{"time":1734148194754,"status":500,"redirect":"https://www.coldcaserecords.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"safetyvideos.gov","name":"Chemical Safety Board","redirect":"https://www.csb.gov/videos/","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.csb.gov/videos/","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":500,"redirect":"https://www.csb.gov/videos","valid":false,"allowed":false,"sitemap":null},{"time":1730174316792,"status":301,"redirect":"https://safetyvideos.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"csb.gov","name":"Chemical Safety Board","redirect":"https://www.csb.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.csb.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":500,"url":"ucia.gov","name":"Central Intelligence Agency","redirect":"http://www.ucia.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"osde.gov","name":"Central Intelligence Agency","redirect":"http://www.osde.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"opensource.gov","name":"Central Intelligence Agency","redirect":"http://www.opensource.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"odci.gov","name":"Central Intelligence Agency","redirect":"http://www.odci.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"istac.gov","name":"Central Intelligence Agency","redirect":"http://www.istac.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"ic.gov","name":"Central Intelligence Agency","redirect":"http://www.ic.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"cia.gov","name":"Central Intelligence Agency","redirect":"https://www.cia.gov/robots.txt","sitemap":"https://www.cia.gov/sitemap/sitemap-0.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"https://www.cia.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1739509031548,"status":404,"redirect":"https://www.cia.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":500,"url":"goldwaterscholarship.gov","name":"Barry Goldwater Scholarship and Excellence in Education Foundation","redirect":"http://www.goldwaterscholarship.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"afrh.gov","name":"Armed Forces Retirement Home","redirect":"http://www.afrh.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":500,"url":"asc.gov","name":"Appraisal Subcommittee","redirect":"http://www.asc.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"http://asc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://asc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"arc.gov","name":"Appalachian Regional Commission","redirect":"https://arc.gov/robots.txt","sitemap":"https://arc.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://arc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://arc.gov/sitemap_index.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"abmc.gov","name":"American Battle Monuments Commission","redirect":"https://www.abmc.gov/robots.txt","sitemap":"https://www.abmc.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.abmc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1742146243896,"status":200,"redirect":"https://www.abmc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":404,"url":"preserveamerica.gov","name":"Advisory Council on Historic Preservation","redirect":"http://www.preserveamerica.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"status":200,"url":"achp.gov","name":"Advisory Council on Historic Preservation","redirect":"https://www.achp.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":404,"redirect":"http://achp.gov/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1729037740707,"status":200,"redirect":"https://www.achp.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1730174316792,"status":500,"redirect":"https://www.achp.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":403,"url":"acus.gov","name":"Administrative Conference of the United States","redirect":"http://www.acus.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.acus.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1739509031548,"status":200,"redirect":"https://www.acus.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"amtrakoig.gov","name":"AMTRAK","redirect":"https://amtrakoig.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://amtrakoig.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"wyo.gov","name":"State of Wyoming","redirect":"https://www.wyo.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.wyo.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"wisconsin.gov","name":"State of Wisconsin","redirect":"https://www.wisconsin.gov/robots.txt","sitemap":"https://www.wisconsin.gov:443/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.wisconsin.gov/Pages/Home.aspx","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"wv.gov","name":"State of West Virginia","redirect":"https://www.wv.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.wv.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"wa.gov","name":"State of Washington","redirect":"https://wa.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://wa.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"virginia.gov","name":"State of Virginia","redirect":"https://www.virginia.gov/robots.txt","sitemap":"https://www.virginia.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.virginia.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.virginia.gov/sitemap.xml"}]},{"status":200,"url":"vermont.gov","name":"State of Vermont","redirect":"https://www.vermont.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.vermont.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"utah.gov","name":"State of Utah","redirect":"https://www.utah.gov/robots.txt","sitemap":"https://www.utah.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.utah.gov/index.html","valid":false,"allowed":false,"sitemap":null},{"time":1728949876846,"status":200,"redirect":"https://www.utah.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.utah.gov/sitemap.xml","source":"robots.txt"},{"time":1730174316792,"status":302,"redirect":"https://utah.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"texas.gov","name":"State of Texas","redirect":"https://www.texas.gov/robots.txt","sitemap":"https://www.texas.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.texas.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.texas.gov/sitemap.xml"}]},{"status":302,"url":"tn.gov","name":"State of Tennessee","redirect":"http://www.tn.gov/","sitemap":null,"valid":false,"allowed":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.tn.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1742146243896,"status":200,"redirect":"https://www.tn.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"sd.gov","name":"State of South Dakota","redirect":"https://www.sd.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://sd.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"sc.gov","name":"State of South Carolina","redirect":"https://sc.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://sc.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ri.gov","name":"State of Rhode Island","redirect":"https://www.ri.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.ri.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"pa.gov","name":"State of Pennsylvania","redirect":"https://pa.gov/robots.txt","sitemap":"https://www.pa.gov/en.sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.pa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.pa.gov/en.sitemap.xml"}]},{"status":200,"url":"oregon.gov","name":"State of Oregon","redirect":"https://www.oregon.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.oregon.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"oklahoma.gov","name":"State of Oklahoma","redirect":"https://oklahoma.gov/robots.txt","sitemap":"https://oklahoma.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://oklahoma.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://oklahoma.gov/sitemap.xml"}]},{"status":404,"url":"ohio.gov","name":"State of Ohio","redirect":"https://www.ohio.gov/","sitemap":null,"valid":false,"allowed":false,"history":[]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nd.gov","name":"State of North Dakota","redirect":"https://nd.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.nd.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nc.gov","name":"State of North Carolina","redirect":"https://www.nc.gov/robots.txt","sitemap":"https://www.nc.gov/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nc.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.nc.gov/sitemap.xml"}]},{"status":200,"url":"ny.gov","name":"State of New York","redirect":"https://www.ny.gov/robots.txt","sitemap":"https://www.ny.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ny.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nm.gov","name":"State of New Mexico","redirect":"https://www.nm.gov/robots.txt","sitemap":"https://nm.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nm.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://nm.gov/sitemap_index.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"nj.gov","name":"State of New Jersey","redirect":"https://nj.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://nj.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"nh.gov","name":"State of New Hampshire","redirect":"https://www.nh.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nh.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"nv.gov","name":"State of Nevada","redirect":"https://nv.gov/robots.txt","sitemap":"https://nv.gov/Sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://nv.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://nv.gov/Sitemap.xml"},{"time":1728949876846,"status":404,"redirect":"http://www.nv.gov/robots.txt","valid":false,"allowed":false,"sitemap":null,"source":false}]},{"status":200,"url":"nebraska.gov","name":"State of Nebraska","redirect":"https://www.nebraska.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.nebraska.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"mt.gov","name":"State of Montana","redirect":"https://mt.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://mt.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"mo.gov","name":"State of Missouri","redirect":"https://www.mo.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.mo.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"mississippi.gov","name":"State of Mississippi","redirect":"https://www.mississippi.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.mississippi.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1729037740707,"status":200,"redirect":"https://www.mississippi.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1730174316792,"status":403,"redirect":"https://www.mississippi.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"mn.gov","name":"State of Minnesota","redirect":"http://validate.perfdrive.com/d5bd5333eafe8b0ccd6023ba818d1aa6/?ssa=778eb54c-f150-4b1a-90ce-b0ed2e8894a8&ssb=12224230096&ssc=https%3A%2F%2Fmn.gov%2Frobots.txt&ssi=ede3d68f-bf56-47f2-96b2-6410d0b29025&ssk=support@shieldsquare.com&ssm=86372016524400940102646112050294&ssn=d06374f2d26f5f7dbb6ffa9530323cc9a60f49ae4824-05c9-471d-bfbb2d&sso=4ce7a24a-b5d03f40d66abe3ff9104df8eff64eddd312e7e38bb6c4a3&ssp=54135309771743833180174382830257193&ssq=33996492661831448689226618311469557163918&ssr=NzUuMjI3LjIxLjIx&sst=CivicHackingAgency/2.0%20ScanGov&ssu=&ssv=&ssw=&ssx=W10=","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1743826667324,"status":500,"redirect":"https://www.mn.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"michigan.gov","name":"State of Michigan","redirect":"https://www.michigan.gov/robots.txt","sitemap":"https://__CdUrl__/sitemap.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.michigan.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.michigan.gov/sitemap.xml"},{"time":1734148194754,"status":200,"redirect":"https://www.michigan.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.michigan.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"mass.gov","name":"State of Massachusetts","redirect":"https://www.mass.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.mass.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"maine.gov","name":"State of Maine","redirect":"https://www.maine.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.maine.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"louisiana.gov","name":"State of Louisiana","redirect":"https://www.louisiana.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.louisiana.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"status":404,"url":"kentucky.gov","name":"State of Kentucky","redirect":"https://www.kentucky.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":404,"redirect":"https://www.kentucky.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"kansas.gov","name":"State of Kansas","redirect":"https://portal.kansas.gov/robots.txt","sitemap":"https://portal.kansas.gov/sitemap_index.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://portal.kansas.gov/","valid":false,"allowed":false,"sitemap":null}]},{"status":200,"url":"iowa.gov","name":"State of Iowa","redirect":"https://www.iowa.gov/robots.txt","sitemap":"https://www.iowa.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.iowa.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.iowa.gov/sitemap.xml"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"in.gov","name":"State of Indiana","redirect":"https://www.in.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.in.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"illinois.gov","name":"State of Illinois","redirect":"https://www.illinois.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.illinois.gov/","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"idaho.gov","name":"State of Idaho","redirect":"https://idaho.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"https://idaho.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://idaho.gov/sitemap.xml"},{"time":1728949876846,"status":200,"redirect":"https://idaho.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://idaho.gov/sitemap.xml","source":"robots.txt"}]},{"status":200,"url":"hawaii.gov","name":"State of Hawaii","redirect":"https://hawaii.gov/robots.txt","sitemap":"https://portal.ehawaii.gov/sitemap.xml","valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://hawaii.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://portal.ehawaii.gov/sitemap.xml"}]},{"status":200,"url":"georgia.gov","name":"State of Georgia","redirect":"https://georgia.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://georgia.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":404,"url":"myflorida.com","name":"State of Florida","redirect":"https://www.myflorida.gov/robots.txt","sitemap":null,"valid":false,"allowed":false,"source":false,"history":[{"time":1726617445612,"status":200,"redirect":"https://www.myflorida.com/robots.txt","valid":false,"allowed":false,"sitemap":null},{"time":1743826667324,"status":200,"redirect":"https://myflorida.com/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"status":200,"url":"de.gov","name":"State of Delaware","redirect":"https://de.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://de.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"ct.gov","name":"State of Connecticut","redirect":"https://ct.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"http://ct.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"colorado.gov","name":"State of Colorado","redirect":"https://www.colorado.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.colorado.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"ca.gov","name":"State of California","redirect":"https://www.ca.gov/robots.txt","sitemap":"https://www.ca.gov/sitemaps/sitemapindex.xml","history":[{"time":1726617445612,"status":200,"redirect":"https://www.ca.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ca.gov/sitemap.xml"},{"time":1734148194754,"status":200,"redirect":"https://www.ca.gov/robots.txt","valid":true,"allowed":true,"sitemap":"https://www.ca.gov/sitemap.xml","source":"robots.txt"},{"time":1737430190371,"status":200,"redirect":"https://ca.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"arkansas.gov","name":"State of Arkansas","redirect":"https://arkansas.gov/robots.txt","sitemap":null,"history":[{"time":1726617445612,"status":200,"redirect":"http://arkansas.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"az.gov","name":"State of Arizona","redirect":"https://az.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://az.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"alaska.gov","name":"State of Alaska","redirect":"https://alaska.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://alaska.gov/robots.txt","valid":true,"allowed":true,"sitemap":null}]},{"status":200,"url":"alabama.gov","name":"State of Alabama","redirect":"https://www.alabama.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.alabama.gov/robots.txt","valid":false,"allowed":false,"sitemap":null}]},{"valid":true,"source":"robots.txt","allowed":true,"status":200,"url":"slc.gov","name":"Salt Lake City, UT","redirect":"https://www.slc.gov/robots.txt","sitemap":"https://www.slc.gov/wp-sitemap.xml","history":[]},{"status":200,"url":"maryland.gov","name":"State of Maryland","redirect":"https://www.maryland.gov/robots.txt","sitemap":null,"valid":true,"allowed":true,"source":"robots.txt","history":[{"time":1726617445612,"status":200,"redirect":"https://www.maryland.gov/robots.txt","valid":true,"allowed":true,"sitemap":null},{"time":1742146243896,"status":200,"redirect":"https://www.maryland.gov/robots.txt","valid":true,"allowed":true,"sitemap":null,"source":"robots.txt"},{"time":1745277859982,"status":500,"redirect":"https://www.maryland.gov/","valid":false,"allowed":false,"sitemap":null}]}]