From 718e957d7346ebd056280a3fe1391681f4e715e3 Mon Sep 17 00:00:00 2001 From: gabestein Date: Thu, 11 Jul 2024 16:01:34 -0400 Subject: [PATCH 01/22] updates bulkexport script to batch exports --- tools/exportCollection.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/exportCollection.js b/tools/exportCollection.js index 31189f2052..03c4679998 100644 --- a/tools/exportCollection.js +++ b/tools/exportCollection.js @@ -8,6 +8,7 @@ import { createPubExportsForLatestRelease } from 'server/export/queries'; import { getPubData } from 'server/rss/queries'; import { promptOkay } from './utils/prompt'; +import { asyncMap } from '../utils/async'; /* Usage: npm run tools exportCollection -- --collectionId=collectionId */ @@ -20,11 +21,14 @@ const { const getPubExports = async (pubId, dest) => { const [pubData] = await getPubData([pubId]); + if (!pubData.releases || pubData.releases.length < 1) { + return Promise.resolve(); + } const pdfUrl = getBestDownloadUrl(pubData, 'pdf'); - const jatsUrl = getBestDownloadUrl(pubData, 'jats'); + const jatsUrl = null; // getBestDownloadUrl(pubData, 'jats'); const finalDest = `${dest}/${pubData.slug}`; - if (!pdfUrl || !jatsUrl) { + if (!pdfUrl /* || !jatsUrl */) { console.log('Missing:', pubData.slug); await createPubExportsForLatestRelease(pubId); await getPubExports(pubId, dest); @@ -66,10 +70,10 @@ const main = async () => { fs.rmdirSync(dest, { recursive: true }); } fs.mkdirSync(dest); - await Promise.all( - collection.collectionPubs.map((collectionPub) => - getPubExports(collectionPub.pubId, dest), - ), + await asyncMap( + collection.collectionPubs, + (collectionPub) => getPubExports(collectionPub.pubId, dest), + { concurrency: 5 }, ); } catch (err) { console.warn(err); From 6d3d1a0723eda0f2b02171bc84229e2738fce90a Mon Sep 17 00:00:00 2001 From: gabestein Date: Thu, 11 Jul 2024 17:21:06 -0400 Subject: [PATCH 02/22] add back jats --- tools/exportCollection.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/exportCollection.js b/tools/exportCollection.js index 03c4679998..45aaec75f6 100644 --- a/tools/exportCollection.js +++ b/tools/exportCollection.js @@ -25,10 +25,10 @@ const getPubExports = async (pubId, dest) => { return Promise.resolve(); } const pdfUrl = getBestDownloadUrl(pubData, 'pdf'); - const jatsUrl = null; // getBestDownloadUrl(pubData, 'jats'); + const jatsUrl = getBestDownloadUrl(pubData, 'jats'); const finalDest = `${dest}/${pubData.slug}`; - if (!pdfUrl /* || !jatsUrl */) { + if (!pdfUrl || !jatsUrl) { console.log('Missing:', pubData.slug); await createPubExportsForLatestRelease(pubId); await getPubExports(pubId, dest); From 7ac5c9890aae9659c2134e50cb41c74d77f853fe Mon Sep 17 00:00:00 2001 From: gabestein Date: Tue, 10 Sep 2024 18:19:34 -0400 Subject: [PATCH 03/22] fix race condition that resulted in no final download --- tools/exportCollection.js | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/tools/exportCollection.js b/tools/exportCollection.js index 45aaec75f6..871f06c6ec 100644 --- a/tools/exportCollection.js +++ b/tools/exportCollection.js @@ -27,25 +27,30 @@ const getPubExports = async (pubId, dest) => { const pdfUrl = getBestDownloadUrl(pubData, 'pdf'); const jatsUrl = getBestDownloadUrl(pubData, 'jats'); const finalDest = `${dest}/${pubData.slug}`; - + const promises = []; if (!pdfUrl || !jatsUrl) { console.log('Missing:', pubData.slug); - await createPubExportsForLatestRelease(pubId); - await getPubExports(pubId, dest); - } else { - fs.mkdirSync(finalDest); - if (jatsUrl) { - await fetch(jatsUrl).then((res) => { - res.body.pipe(fs.createWriteStream(`${finalDest}/${pubData.slug}.xml`)); - }); - } - if (pdfUrl) { - await fetch(pdfUrl).then((res) => { - res.body.pipe(fs.createWriteStream(`${finalDest}/${pubData.slug}.pdf`)); - }); - } + promises.push(createPubExportsForLatestRelease(pubId)); + promises.push(getPubExports(pubId, dest)); + console.log('resolve recursive call', promises); + return Promise.all(promises); } - return Promise.resolve(); + // Make the empty dir + fs.mkdirSync(finalDest); + promises.push( + fetch(jatsUrl).then(async (res) => { + console.log('getting JATS...', jatsUrl); + res.body.pipe(fs.createWriteStream(`${finalDest}/${pubData.slug}.xml`)); + }), + ); + promises.push( + fetch(pdfUrl).then(async (res) => { + console.log('getting PDF...', pdfUrl); + res.body.pipe(fs.createWriteStream(`${finalDest}/${pubData.slug}.pdf`)); + }), + ); + console.log('resolving internal loop...', promises); + return Promise.all(promises); }; const main = async () => { From fa081c018b4684ba50533c23fb0e4de3cc4b59f8 Mon Sep 17 00:00:00 2001 From: gabestein Date: Fri, 20 Sep 2024 11:18:09 -0400 Subject: [PATCH 04/22] still not perfect but better --- tools/exportCollection.js | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tools/exportCollection.js b/tools/exportCollection.js index 871f06c6ec..78036aa091 100644 --- a/tools/exportCollection.js +++ b/tools/exportCollection.js @@ -37,20 +37,18 @@ const getPubExports = async (pubId, dest) => { } // Make the empty dir fs.mkdirSync(finalDest); - promises.push( - fetch(jatsUrl).then(async (res) => { - console.log('getting JATS...', jatsUrl); - res.body.pipe(fs.createWriteStream(`${finalDest}/${pubData.slug}.xml`)); - }), - ); - promises.push( - fetch(pdfUrl).then(async (res) => { - console.log('getting PDF...', pdfUrl); - res.body.pipe(fs.createWriteStream(`${finalDest}/${pubData.slug}.pdf`)); - }), - ); - console.log('resolving internal loop...', promises); - return Promise.all(promises); + try { + const jats = await fetch(jatsUrl); + console.log('getting JATS...', jatsUrl); + jats.body.pipe(fs.createWriteStream(`${finalDest}/${pubData.slug}.xml`)); + const pdf = await fetch(pdfUrl); + console.log('getting PDF...', pdfUrl); + pdf.body.pipe(fs.createWriteStream(`${finalDest}/${pubData.slug}.pdf`)); + console.log('resolving promise...'); + return Promise.resolve(); + } catch (error) { + throw new Error(error); + } }; const main = async () => { From 31a8af98545e6b2370f70db29f7a9a9e7e5f135b Mon Sep 17 00:00:00 2001 From: Gabriel Stein Date: Wed, 17 Jul 2024 17:11:25 -0400 Subject: [PATCH 05/22] update: Langing page for Platform (#3129) * redo landing page and remove some old routes * get rid of landing page lint errors * remove community creation links * require superadmin to create community * update links to kf.org * Adjust jumbotron height * update footer link * fix tests * better test fix * ok actually do real tests fine gabe does tod now * sigh spell it right * I love tests --------- Co-authored-by: Travis Rich --- client/components/Footer/Footer.tsx | 15 +- .../GlobalControls/GlobalControls.tsx | 7 +- client/containers/About/About.tsx | 4 - .../CommunityCreate/CommunityCreate.tsx | 26 +- client/containers/Explore/Explore.tsx | 11 +- client/containers/Landing/Landing.tsx | 361 +----------------- client/containers/Landing/landing.scss | 2 +- server/community/__tests__/api.test.ts | 27 +- server/community/api.ts | 3 +- server/routes/index.ts | 8 +- 10 files changed, 61 insertions(+), 403 deletions(-) diff --git a/client/components/Footer/Footer.tsx b/client/components/Footer/Footer.tsx index 08e2d4b18a..c56ece1d10 100644 --- a/client/components/Footer/Footer.tsx +++ b/client/components/Footer/Footer.tsx @@ -33,7 +33,6 @@ const defaultProps = { type Props = OwnProps & typeof defaultProps; const basePubPubFooterLinks = [ - { id: '1', title: 'Create your community', href: '/community/create' }, { id: '2', title: 'Login', href: '/login' }, { id: '3', title: 'Signup', href: '/signup' }, { id: '4', title: 'Legal', href: '/legal' }, @@ -143,21 +142,17 @@ const Footer = (props: Props) => {
diff --git a/client/components/GlobalControls/GlobalControls.tsx b/client/components/GlobalControls/GlobalControls.tsx index cf2ddab065..f1633ad895 100644 --- a/client/components/GlobalControls/GlobalControls.tsx +++ b/client/components/GlobalControls/GlobalControls.tsx @@ -99,9 +99,10 @@ const GlobalControls = (props: Props) => { if (isBasePubPub) { return ( <> - - - + {renderSearch()} ); diff --git a/client/containers/About/About.tsx b/client/containers/About/About.tsx index dcb5c9b516..90adf8338f 100644 --- a/client/containers/About/About.tsx +++ b/client/containers/About/About.tsx @@ -19,10 +19,6 @@ const About = function () { from drafting documents, conducting peer review, and hosting entire journal and book websites to collecting and displaying reader feedback and analytics.

-

- You can get started now by{' '} - creating your community. -

Features, Benefits, and Tradeoffs

PubPub is an open-source, hosted, free-to-use content management system designed diff --git a/client/containers/CommunityCreate/CommunityCreate.tsx b/client/containers/CommunityCreate/CommunityCreate.tsx index 162ac5f380..7715a1ad9f 100644 --- a/client/containers/CommunityCreate/CommunityCreate.tsx +++ b/client/containers/CommunityCreate/CommunityCreate.tsx @@ -82,15 +82,16 @@ const CommunityCreate = () => {

Create Community

- PubPub and all its features are free to use.* If you value - PubPub, we ask you to consider supporting our work by becoming a member - of the Knowledge Futures Group.{' '} + PubPub is evolving, and we are currently only allowing new community + creation for existing users with an explicit short-term need. Learn more + by reading our announcement. If you are an existing user who needs to + create a community, please{' '} - Learn more + get in touch .

@@ -178,21 +179,6 @@ const CommunityCreate = () => { /> -

- * We limit DOI registrations to 10 per community per year, if - published via PubPub's Crossref membership. Once the limit is reached, - we ask that you become a{' '} - - KFG member - - , at any level, and allow us to pass on the Crossref fee of $1 per DOI - registered. For groups with their own Crossref membership, there is no - additional fee for creating or depositing DOIs. -

)} diff --git a/client/containers/Explore/Explore.tsx b/client/containers/Explore/Explore.tsx index dc51aee1c5..b0c231ec11 100644 --- a/client/containers/Explore/Explore.tsx +++ b/client/containers/Explore/Explore.tsx @@ -17,13 +17,10 @@ const Explore = (props: Props) => {

Explore communities

- PubPub hosts over 3,000 communities, with more added every day! Anyone - can create a free PubPub community at - any time; all you need is an account and some very basic community - details. Below are 40 PubPub communities that we think form a good - snapshot of what you can create with PubPub's spaces and features. We - hope they provide inspiration for your own spaces, designs, and - workflows. + PubPub hosts over 3,000 communities, with more added every day! Below + are 40 PubPub communities that we think form a good snapshot of what you + can create with PubPub's spaces and features. We hope they provide + inspiration for your own spaces, designs, and workflows.
diff --git a/client/containers/Landing/Landing.tsx b/client/containers/Landing/Landing.tsx index 9dd986af8f..b82bac6ebe 100644 --- a/client/containers/Landing/Landing.tsx +++ b/client/containers/Landing/Landing.tsx @@ -1,165 +1,9 @@ import React from 'react'; import { Classes } from '@blueprintjs/core'; -import { Icon } from 'components'; - require('./landing.scss'); -const features = [ - { - icon: 'badge', - title: 'DOI Support', - desc: 'Generate CrossRef DOIs for your documents in one click.', - }, - { - icon: 'shield', - title: 'Submissions & Review', - desc: 'Manage submissions and peer review directly on PubPub.', - }, - { - icon: 'comment', - title: 'Discussions & Annotations', - desc: 'Host public and private discussions with your readers and community, whether in your classroom or across the world.', - }, - { - icon: 'page-layout', - title: 'Easily Customizable Layouts', - desc: 'Create your custom site without writing a line of code.', - }, - { - icon: 'book', - title: 'Collection Metadata', - desc: 'Include article & collection-level metadata for easier organization of content and improved discovery.', - }, - { - icon: 'people', - title: 'Access Control', - desc: 'Allow anyone to access your content, or just the people you choose.', - }, - { - icon: 'grouped-bar-chart', - title: 'Impact Measurement', - desc: 'Learn about the people visiting your community with a full suite of privacy-respecting analytics.', - }, - { - icon: 'graph', - title: 'Content Connections', - desc: 'Add typed relationships — reviews, commentary, supplement, etc. — to your content and deposit them to Crossref.', - }, - { - icon: 'export', - title: 'Document Export', - desc: 'Export your work to PDF, Word, Markdown, LaTeX, JATS XML, and more.', - }, -] as const; - -const communities = [ - { - name: 'Harvard Data Science Review', - description: 'A microscopic, telescopic & kaleidoscopic view of data science.', - logo: '/static/landing/hdsr.png', - type: 'Journals', - category: 'Science', - link: 'https://hdsr.mitpress.mit.edu', - }, - { - name: 'Frankenbook', - description: - 'A collaborative, multimedia reading experiment with Mary Shelley’s classic novel.', - logo: '/static/landing/frankenbook.png', - type: 'Books', - category: 'Literature', - link: 'https://frankenbook.org', - }, - { - name: 'Collective Wisdom', - description: "The British Library's early access and community review site.", - logo: '/static/landing/collective.png', - type: 'Resources', - category: 'Reports', - link: 'https://britishlibrary.pubpub.org/', - }, - { - name: 'SERC Ethical Computing', - description: 'A series on the social and ethical responsibilities of computing.', - logo: '/static/landing/serc.png', - type: 'Resources', - category: 'Case studies', - link: 'https://mit-serc.pubpub.org', - }, - { - name: 'Contours Collaborations', - description: 'A series on the social and ethical responsibilities of computing.', - logo: '/static/landing/contours.png', - type: 'Exhibits', - category: 'Arts', - link: 'https://contours.pubpub.org', - }, - { - name: 'Fermentology', - description: 'On the culture, history, and novelty of fermented things.', - logo: '/static/landing/fermentology.png', - type: 'Learning Series', - category: 'Multimedia', - link: 'https://fermentology.pubpub.org/', - }, - { - name: 'Grad Journal of Food Studies', - description: - 'An international student-run and refereed platform dedicated to encouraging and promoting interdisciplinary food scholarship at the graduate level', - logo: '/static/landing/gafs.png', - type: 'Student Journals', - category: 'Interdisciplinary', - link: 'https://gradfoodstudies.pubpub.org/', - }, - { - name: 'Critical Distance', - description: 'A pandemic and games essay jam.', - logo: '/static/landing/pgej.png', - type: 'Conferences', - category: 'Sprint', - link: 'https://pandemics-and-games-essay-jam.pubpub.org/', - }, - { - name: 'MIT Press Open Architecture', - description: 'An open collection of out-of-print humanities books.', - logo: '/static/landing/arch.png', - type: 'Collections', - category: 'Open humanities', - link: 'https://mitp-arch.mitpress.mit.edu/', - }, -]; - const Landing = () => { - const featureGrid = features.map((feature) => { - return ( -
- -
-

{feature.title}

-

{feature.desc}

-
-
- ); - }); - - const communityGrid = communities.map((community) => { - return ( - // @ts-expect-error ts-migrate(2322) FIXME: Type '{ children: Element[]; className: string; ke... Remove this comment to see the full error message -
-
- {community.type} / {community.category} -
-

- {community.name} -

- - {`Logo - -

{community.description}

-
- ); - }); return (