commoncrawl (Common Crawl Foundation)

commoncrawl (Common Crawl Foundation) Common Crawl</h1>\n<p>Welcome to the Common Crawl Foundation's Hugging Face page!</p>\n<p>We aim to provide metadata and experimental versions of our latest data products here.</p>\n<h3>Useful Links</h3>\n<ul>\n<li><a href=\"https://commoncrawl.org/\" rel=\"nofollow\">Common Crawl's official website</a></li>\n<li><a href=\"https://commoncrawl.github.io/cc-crawl-statistics/\" rel=\"nofollow\">Our existing statistics webpages</a> (<a href=\"https://github.com/commoncrawl/cc-crawl-statistics\" rel=\"nofollow\">GitHub repo</a>)</li>\n<li><a href=\"https://status.commoncrawl.org/\" rel=\"nofollow\">AWS infrastructure status page</a></li>\n</ul>\n<h3>Datasets</h3>\n<p>Explore our datasets hosted on Hugging Face:</p>\n<ul>\n<li><a href=\"https://huggingface.co/datasets/commoncrawl/citations\" rel=\"nofollow\">Common Crawl Citations</a></li>\n<li><a href=\"https://huggingface.co/datasets/commoncrawl/citations-annotated\" rel=\"nofollow\">Common Crawl Citations, Annotated</a></li>\n<li><a href=\"https://huggingface.co/datasets/commoncrawl/statistics\" rel=\"nofollow\">Common Crawl Statistics</a></li>\n<li><a href=\"https://huggingface.co/datasets/commoncrawl/eot2024_hostlevel_logs\" rel=\"nofollow\">EOT 2024 Host-Level Logs</a> (only available to EOT collaborators)</li>\n</ul>\n<p>We look forward to supporting the research and development community with these resources.</p>\n","classNames":"hf-sanitized hf-sanitized-zgMenJn7M1sgnA8pdfTei"},"users":[{"_id":"656f69ecec26abeb3cf6e3f9","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/noauth/dq6qLDpKuvuNkk9fnO7lS.png","isPro":false,"fullname":"Thom Vaughan","user":"tvaughan","type":"user"},{"_id":"602705869cad998710e2e8ab","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1632322117476-602705869cad998710e2e8ab.jpeg","isPro":false,"fullname":"Pedro Ortiz Suarez","user":"pjox","type":"user"},{"_id":"656f9eace2c3369baa436f63","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/656f9eace2c3369baa436f63/9kDUGrrBnY9o1x0zoIkzd.jpeg","isPro":false,"fullname":"Paul Lazar","user":"suneecat","type":"user"},{"_id":"656fbeae7734a829bbd16252","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/656fbeae7734a829bbd16252/s0_NEIevmFM3dncq0gHPn.jpeg","isPro":false,"fullname":"Greg Lindahl","user":"greglindahl","type":"user"},{"_id":"6643b244f4e371aaf4712608","avatarUrl":"/avatars/a283266a917c3078ff371b0fffa12ca7.svg","isPro":false,"fullname":"Ford H","user":"fheilz","type":"user"},{"_id":"664b8fcec54cd3b3297c80c7","avatarUrl":"/avatars/60ed2bc9e1b30d1f6dd0881a43e24465.svg","isPro":false,"fullname":"Jen English","user":"jenglish-cc","type":"user"},{"_id":"617a66b5ff689804c90eca3a","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/617a66b5ff689804c90eca3a/X4aHDNCut7X-Gh6BV0zD-.jpeg","isPro":false,"fullname":"Sebastian Nagel","user":"snagel","type":"user"},{"_id":"60ed4de630236d25bc1bfb4b","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1628610761629-60ed4de630236d25bc1bfb4b.png","isPro":false,"fullname":"Laurie Burchell","user":"laurievb","type":"user"},{"_id":"649a9cbb4952ac2c7a205444","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/649a9cbb4952ac2c7a205444/2bk-u1TlTn0k8OI6RzGmZ.png","isPro":false,"fullname":"Hande Celikkanat","user":"handecelikkanat","type":"user"},{"_id":"5efda656ff69163f6f59e5d2","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/5efda656ff69163f6f59e5d2/YJM48t7eddfun1E3k1iNS.jpeg","isPro":false,"fullname":"malteos","user":"malteos","type":"user"},{"_id":"6890b6412ce18b0d9e72f665","avatarUrl":"/avatars/8aeea756dde02e36e1176b5b317b3189.svg","isPro":false,"fullname":"Thijs Dalhuijsen","user":"dalhuijsen","type":"user"},{"_id":"6177bd3248ce96fe95f80e0e","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1635237156064-noauth.png","isPro":false,"fullname":"Luca","user":"lfoppiano","type":"user"},{"_id":"6384d424ac61472e5e92d403","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6384d424ac61472e5e92d403/RfEyLD3FmEobLcpNE9pNu.png","isPro":false,"fullname":"Catherine Arnett","user":"catherinearnett","type":"user"},{"_id":"6a13445abe32e5ae525379c3","avatarUrl":"/avatars/c3e94576f59cee21856976b0cc015b1b.svg","isPro":false,"fullname":"Michael Paris","user":"parismic","type":"user"}],"userCount":14,"collections":[],"datasets":[{"author":"commoncrawl","downloads":647,"gated":false,"id":"commoncrawl/statistics","lastModified":"2026-07-24T11:40:50.000Z","datasetsServerInfo":{"viewer":"viewer","numRows":642081,"libraries":[],"formats":[],"modalities":["tabular","text"]},"private":false,"repoType":"dataset","likes":29,"isLikedByUser":false,"isBenchmark":false,"isTraces":false},{"author":"commoncrawl","downloads":210,"gated":"auto","id":"commoncrawl/CommonLID","lastModified":"2026-07-05T16:02:46.000Z","datasetsServerInfo":{"viewer":"viewer","numRows":373230,"libraries":["datasets","dask","polars","mlcroissant"],"formats":["json"],"modalities":["text"]},"private":false,"repoType":"dataset","likes":55,"isLikedByUser":false,"isBenchmark":false,"isTraces":false},{"author":"commoncrawl","downloads":257,"gated":false,"id":"commoncrawl/commonlid-results","lastModified":"2026-06-10T08:54:48.000Z","datasetsServerInfo":{"viewer":"preview","numRows":0,"libraries":[],"formats":[],"modalities":[]},"private":false,"repoType":"dataset","likes":2,"isLikedByUser":false,"isBenchmark":false,"isTraces":false},{"author":"commoncrawl","downloads":56,"gated":false,"id":"commoncrawl/web-graph-testing-v1","lastModified":"2026-05-28T12:54:23.000Z","private":false,"repoType":"dataset","likes":0,"isLikedByUser":false,"isBenchmark":false,"isTraces":false},{"author":"commoncrawl","downloads":293,"gated":false,"id":"commoncrawl/citations","lastModified":"2026-04-02T13:42:38.000Z","datasetsServerInfo":{"viewer":"viewer","numRows":9180,"libraries":["datasets","pandas","mlcroissant","polars"],"formats":["json"],"modalities":["text"]},"private":false,"repoType":"dataset","likes":4,"isLikedByUser":false,"isBenchmark":false,"isTraces":false},{"author":"commoncrawl","downloads":368,"gated":false,"id":"commoncrawl/gneissweb-annotation-host-testing-v1","lastModified":"2025-12-11T18:07:12.000Z","datasetsServerInfo":{"viewer":"viewer","numRows":617167171,"libraries":["datasets","dask","mlcroissant","polars"],"formats":["parquet"],"modalities":["tabular","text"]},"private":false,"repoType":"dataset","likes":1,"isLikedByUser":false,"isBenchmark":false,"isTraces":false},{"author":"commoncrawl","downloads":10208,"gated":false,"id":"commoncrawl/gneissweb-annotation-url-testing-v1","lastModified":"2025-12-10T22:41:31.000Z","datasetsServerInfo":{"viewer":"viewer","numRows":11541703731,"libraries":["datasets","dask","mlcroissant","polars"],"formats":["parquet"],"modalities":["tabular","text"]},"private":false,"repoType":"dataset","likes":0,"isLikedByUser":false,"isBenchmark":false,"isTraces":false},{"author":"commoncrawl","downloads":6787,"gated":false,"id":"commoncrawl/host-index-testing-v2","lastModified":"2025-11-10T16:32:16.000Z","datasetsServerInfo":{"viewer":"preview","numRows":0,"libraries":[],"formats":[],"modalities":[]},"private":false,"repoType":"dataset","likes":0,"isLikedByUser":false,"isBenchmark":false,"isTraces":false},{"author":"commoncrawl","downloads":4,"gated":"manual","id":"commoncrawl/eot2024_hostlevel_logs","lastModified":"2024-10-09T02:11:09.000Z","datasetsServerInfo":{"viewer":"viewer","numRows":271322,"libraries":["datasets","pandas","mlcroissant","polars"],"formats":["parquet"],"modalities":["tabular","text"]},"private":false,"repoType":"dataset","likes":1,"isLikedByUser":false,"isBenchmark":false,"isTraces":false}],"models":[],"paperPreviews":[],"spaces":[{"author":"commoncrawl","authorData":{"_id":"656f94d2e2c3369baa4167b0","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/656fbeae7734a829bbd16252/RANGDo_aKsWYL2nKwkISh.png","fullname":"Common Crawl Foundation","name":"commoncrawl","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"plan":"team","followerCount":278,"isUserFollowing":false},"colorFrom":"indigo","colorTo":"pink","createdAt":"2026-05-06T07:28:04.000Z","disabled":false,"emoji":"🌍","id":"commoncrawl/commonlid","lastModified":"2026-08-17T11:22:22.000Z","likes":4,"pinned":false,"private":false,"sdk":"gradio","repoType":"space","runtime":{"stage":"RUNNING","hardware":{"current":"cpu-basic","requested":"cpu-basic"},"gcTimeout":172800,"replicas":{"current":1,"requested":1},"devMode":false,"domains":[{"domain":"commoncrawl-commonlid.hf.space","stage":"READY"}],"sha":"f3edf4da791fd3837479d63f97da84a27d2b14b7"},"title":"CommonLID Leaderboard","isLikedByUser":false,"ai_short_description":"Explore the CommonLID language ID leaderboard online","ai_category":"Data Visualization","trendingScore":0,"tags":["gradio","region:us"],"featured":false,"visibility":"public"},{"author":"commoncrawl","authorData":{"_id":"656f94d2e2c3369baa4167b0","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/656fbeae7734a829bbd16252/RANGDo_aKsWYL2nKwkISh.png","fullname":"Common Crawl Foundation","name":"commoncrawl","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"plan":"team","followerCount":278,"isUserFollowing":false},"colorFrom":"blue","colorTo":"gray","createdAt":"2025-12-08T11:59:31.000Z","disabled":false,"emoji":"📜","id":"commoncrawl/cc-citations","lastModified":"2026-02-19T11:49:08.000Z","likes":9,"pinned":false,"private":false,"sdk":"static","repoType":"space","runtime":{"stage":"RUNNING","hardware":{"current":null,"requested":null},"replicas":{"requested":1,"current":1}},"shortDescription":"Scientific articles using or citing Common Crawl data","title":"cc-citations","isLikedByUser":false,"ai_short_description":"Explore Common Crawl research citations on an interactive map","ai_category":"Data Visualization","trendingScore":0,"tags":["static","region:us"],"featured":false,"visibility":"public"}],"buckets":[{"_id":"6a553b180beae73afa6cd886","id":"commoncrawl/warc2zip-examples","author":"commoncrawl","private":false,"createdAt":"2026-07-13T19:23:04.000Z","updatedAt":"2026-07-13T19:39:34.920Z","size":1507217695,"totalFiles":3,"repoType":"bucket","cdnRegions":[]},{"_id":"69dc9d04721d8a245175e4e1","id":"commoncrawl/commoncrawl","author":"commoncrawl","private":false,"createdAt":"2026-04-13T07:36:36.000Z","updatedAt":"2026-06-30T08:06:00.052Z","size":355601823812182,"totalFiles":1502729,"repoType":"bucket","cdnRegions":[{"provider":"gcp","region":"us"},{"provider":"gcp","region":"eu"},{"provider":"aws","region":"eu"}]},{"_id":"699eb0ac01ea68738e16d686","id":"commoncrawl/test-bucket","author":"commoncrawl","private":false,"createdAt":"2026-02-25T08:19:56.000Z","updatedAt":"2026-04-28T08:17:54.490Z","size":116616729,"totalFiles":12,"repoType":"bucket","cdnRegions":[{"provider":"gcp","region":"us"},{"provider":"gcp","region":"eu"},{"provider":"aws","region":"us"},{"provider":"aws","region":"eu"}]}],"numBuckets":3,"kernels":[],"numKernels":0,"numDatasets":9,"numModels":0,"numSpaces":3,"lastOrgActivities":[{"time":"2026-08-17T11:22:23.112Z","user":"malteos","userAvatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/5efda656ff69163f6f59e5d2/YJM48t7eddfun1E3k1iNS.jpeg","org":"commoncrawl","orgAvatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/656fbeae7734a829bbd16252/RANGDo_aKsWYL2nKwkISh.png","type":"discussion","discussionData":{"num":1,"author":{"_id":"5efda656ff69163f6f59e5d2","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/5efda656ff69163f6f59e5d2/YJM48t7eddfun1E3k1iNS.jpeg","fullname":"malteos","name":"malteos","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":74,"isUserFollowing":false},"repo":{"name":"commoncrawl/commonlid","type":"space"},"title":"Bump Gradio sdk_version 5.9.1 -> 6.14.0","status":"merged"…