diff --git a/Dockerfile b/Dockerfile index a1a4299fc..3483200b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,38 +16,10 @@ COPY . . RUN npm run build # production environment -FROM alpine:latest -RUN apk update && apk add --no-cache \ - build-base \ - pcre-dev \ - zlib-dev \ - openssl-dev \ - git \ - wget \ - apache2-utils -WORKDIR /usr/src -RUN wget http://nginx.org/download/nginx-1.28.0.tar.gz && \ - tar -zxvf nginx-1.28.0.tar.gz && \ - git clone https://github.com/atomx/nginx-http-auth-digest.git -WORKDIR /usr/src/nginx-1.28.0 -RUN ./configure \ - --prefix=/etc/nginx \ - --sbin-path=/usr/sbin/nginx \ - --conf-path=/etc/nginx/nginx.conf \ - --error-log-path=/dev/stderr \ - --http-log-path=/dev/stdout \ - --with-http_ssl_module \ - --with-http_v2_module \ - --with-http_gzip_static_module \ - --with-http_realip_module \ - --add-module=/usr/src/nginx-http-auth-digest && \ - make && \ - make install +FROM nginx:stable COPY --from=build /app/build /usr/share/nginx/html -COPY ./nginx/templates/default.conf.template /etc/nginx/conf.d/default.conf -COPY ./nginx/.htdigest /etc/nginx/.htdigest +COPY ./nginx/templates /etc/nginx/templates COPY ./nginx/nginx.conf /etc/nginx/nginx.conf COPY ./nginx/server.crt /etc/nginx/server.crt COPY ./nginx/server.key /etc/nginx/server.key EXPOSE 3000 -CMD ["/usr/sbin/nginx", "-g", "daemon off;"] \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf index c14e615e7..c76adf6e2 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -1,7 +1,7 @@ worker_processes auto; events { - worker_connections 1024; + worker_connections 4096; } http { @@ -10,7 +10,12 @@ http { sendfile on; keepalive_timeout 65; - + log_format main '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent" ' + '"$http_x_forwarded_for" ' + '$request_time $upstream_response_time'; + access_log /dev/stdout main; include /etc/nginx/conf.d/*.conf; server_tokens off; } \ No newline at end of file diff --git a/nginx/templates/default.conf.template b/nginx/templates/default.conf.template index 11d1811ea..59828510c 100644 --- a/nginx/templates/default.conf.template +++ b/nginx/templates/default.conf.template @@ -1,3 +1,11 @@ +upstream es_cluster { + least_conn; + server 136.187.110.96:9200 max_fails=3 fail_timeout=30s; + server 136.187.110.96:9201 max_fails=3 fail_timeout=30s; + server 136.187.110.96:9202 max_fails=3 fail_timeout=30s; + + keepalive 32; +} server { listen 3000 ssl; server_name localhost osm.ir.rcos.nii.ac.jp 136.187.110.114; @@ -10,18 +18,12 @@ server { real_ip_recursive on; location / { - auth_digest "josm"; - auth_digest_user_file /etc/nginx/.htdigest; - auth_digest_expires 1800s; - auth_digest_replays 1000; - root /usr/share/nginx/html; index index.html index.htm; try_files $uri $uri/ /index.html =404; } location /health { default_type text/plain; - access_log off; return 200; } location /mailer/ { @@ -36,7 +38,10 @@ server { rewrite /elasticsearch/(.*) /$1 break; proxy_set_header Authorization "ApiKey XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"; proxy_ssl_verify off; - proxy_pass https://136.187.110.96:9200; + proxy_pass https://es_cluster; + + proxy_http_version 1.1; + proxy_set_header Connection ""; add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; diff --git a/public/documents/privacypolicy_en.pdf b/public/documents/privacypolicy_en.pdf new file mode 100644 index 000000000..6ad774bed Binary files /dev/null and b/public/documents/privacypolicy_en.pdf differ diff --git a/public/documents/privacypolicy_ja.pdf b/public/documents/privacypolicy_ja.pdf new file mode 100644 index 000000000..89a54bbd2 Binary files /dev/null and b/public/documents/privacypolicy_ja.pdf differ diff --git a/public/documents/terms_en.pdf b/public/documents/terms_en.pdf new file mode 100644 index 000000000..4afaf8a0f Binary files /dev/null and b/public/documents/terms_en.pdf differ diff --git a/public/documents/terms_ja.pdf b/public/documents/terms_ja.pdf new file mode 100644 index 000000000..2d8bda966 Binary files /dev/null and b/public/documents/terms_ja.pdf differ diff --git a/src/components/Charts/publications/archives/dynamique-hal/get-data-josm.js b/src/components/Charts/publications/archives/dynamique-hal/get-data-josm.js index 694ae1c4f..0d76bd55e 100644 --- a/src/components/Charts/publications/archives/dynamique-hal/get-data-josm.js +++ b/src/components/Charts/publications/archives/dynamique-hal/get-data-josm.js @@ -3,239 +3,198 @@ import { useEffect, useState } from 'react'; import { useIntl } from 'react-intl'; // import { ES_API_URL, HEADERS } from '../../../../../config/config'; -import { ES_API_URL, IS_TEST } from '../../../../../config/config'; +import { ES_API_URL } from '../../../../../config/config'; // import getFetchOptions from '../../../../../utils/chartFetchOptions'; import { - capitalize, - getCSSValue, - getObservationLabel, + capitalize, + getCSSValue, } from '../../../../../utils/helpers'; -function useGetData(beforeLastObservationSnap, lastObservationSnap, domain) { - const intl = useIntl(); - const [data, setData] = useState([]); - const [isLoading, setLoading] = useState(true); - const [isError, setError] = useState(false); - const bsoDomain = intl.formatMessage({ id: `app.bsoDomain.${domain}` }); - - async function GetData() { - // 1回目のクエリ 最新のcalc_dateを取得 - const latestDateRes = await Axios.post(ES_API_URL, { - size: 1, - _source: ['calc_date'], - sort: [ - { - calc_date: { - order: 'desc', - }, - }, - ], - query: { - term: { - data_type: 'archives.dynamique-hal.get-data', - }, - }, - }); - - // 1回目のクエリで得たcalc_dateをlatestCalcDateに代入 - /* eslint-disable no-underscore-dangle */ - const latestCalcDate = latestDateRes.data.hits.hits[0]._source.calc_date; - - // 2回目のクエリ 最新のcalc_dateのデータを取得 - const preRes = await Axios.post(ES_API_URL, { - query: { - bool: { - must: [ - { term: { calc_date: latestCalcDate } }, - { term: { data_type: 'archives.dynamique-hal.get-data' } }, - ], - }, - }, - }); - - let res; - if (lastObservationSnap) { - // 成形処理 - /* eslint-disable no-underscore-dangle */ - res = [ - { - data: { - aggregations: { - by_publication_year: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: preRes.data.hits.hits[0]._source.data.map((item) => ({ - key: item.publication_year, - doc_count: item.jp_repo, - })), +function useGetDataTest(beforeLastObservationSnap, lastObservationSnap, domain) { + const [data, setData] = useState({}); + const [isLoading, setLoading] = useState(true); + const [isError, setError] = useState(false); + const intl = useIntl(); + const bsoDomain = intl.formatMessage({ id: `app.bsoDomain.${domain}` }); + + async function getDataByObservationSnaps() { + // 1回目のクエリ 最新のcalc_dateを取得 + const latestDateRes = await Axios.post(ES_API_URL, { + size: 0, + aggs: { + unique_calc_dates: { + terms: { + field: 'calc_date', + size: 10000, + }, + }, }, - }, - }, - }, - { - data: { - aggregations: { - by_publication_year: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: preRes.data.hits.hits[0]._source.data.map((item) => ({ - key: item.publication_year, - doc_count: item.other_repo + item.jp_repo, - })), + query: { + term: { + data_type: 'archives.dynamique-ouverture.get-data', + }, }, - }, - }, - }, - ]; - } else { - res = [{ - data: { - aggregations: { - by_publication_year: { - buckets: [], - }, - }, - }, - }, - { - data: { - aggregations: { - by_publication_year: { - buckets: [], - }, - }, - }, - }]; - } - /* eslint-enable no-underscore-dangle */ - if (IS_TEST) { - console.log('dynamique-hal_preRes:', preRes); // eslint-disable-line no-console - console.log('dynamique-hal_res:', res); // eslint-disable-line no-console - } + }); - let dataHAL = res[0].data.aggregations.by_publication_year.buckets; - dataHAL = dataHAL - .sort((a, b) => a.key - b.key) - .filter( - (el) => el.key > 2012 - && parseInt(el.key, 10) - < parseInt(lastObservationSnap.substring(0, 4), 10), - ); - let dataArchive = res[1].data.aggregations.by_publication_year.buckets; - dataArchive = dataArchive - .sort((a, b) => a.key - b.key) - .filter( - (el) => el.key > 2012 - && parseInt(el.key, 10) - < parseInt(lastObservationSnap.substring(0, 4), 10), - ); - const publicationYears = []; - const hal = []; - const notHal = []; - dataHAL.forEach((el, index) => { - publicationYears.push(el.key); - hal.push({ - y_abs: el.doc_count, - y: (100 * el.doc_count) / dataArchive[index].doc_count, - bsoDomain, - y_percHAL: (100 * el.doc_count) / dataArchive[index].doc_count, - y_tot: dataArchive[index].doc_count, - x: el.key, - }); - notHal.push({ - y_abs: dataArchive[index].doc_count - el.doc_count, - y: - (100 * (dataArchive[index].doc_count - el.doc_count)) - / dataArchive[index].doc_count, - bsoDomain, - y_percHAL: - (100 * (dataArchive[index].doc_count - el.doc_count)) - / dataArchive[index].doc_count, - y_tot: dataArchive[index].doc_count, - x: el.key, - }); - }); - const dataGraph2 = [ - { - name: capitalize( - intl.formatMessage({ - id: 'app.health-publi.repositories.dynamique-hal.notHal', - }), - ), - data: notHal, - color: getCSSValue('--green-medium-150'), - }, - { - name: capitalize( - intl.formatMessage({ - id: 'app.health-publi.repositories.dynamique-hal.hal', - }), - ), - data: hal, - color: getCSSValue('--acces-ouvert'), - }, - ]; - - const valueHalLabel = capitalize( - intl.formatMessage({ - id: 'app.health-publi.repositories.dynamique-hal.hal', - }), - ); - const valueNotHalLabel = capitalize( - intl.formatMessage({ - id: 'app.health-publi.repositories.dynamique-hal.notHal', - }), - ); - let valueHAL = ''; - let valueNotHAL = ''; - if (dataGraph2) { - valueHAL = dataGraph2 - .find((item) => item.name === valueHalLabel) - ?.data.find( - (item) => item.x.toString() - === getObservationLabel(beforeLastObservationSnap, intl), - ) - ?.y.toFixed(0); - valueNotHAL = dataGraph2 - .find((item) => item.name === valueNotHalLabel) - ?.data.find( - (item) => item.x.toString() - === getObservationLabel(beforeLastObservationSnap, intl), - ) - ?.y.toFixed(0); - } + // ユニークな `calc_date` のリストを取得 + const yearMonthDayList = latestDateRes.data.aggregations.unique_calc_dates.buckets.map( + (bucket) => bucket.key_as_string.slice(0, 10), + ); + + const yearGroups = yearMonthDayList.reduce((acc, date) => { + const year = date.slice(0, 4); + if (!acc[year]) acc[year] = []; + acc[year].push(date); + return acc; + }, {}); + const lastDateOfYear = []; + + // 各年のデータから最終日を取得 + Object.keys(yearGroups).forEach((year) => { + /* eslint-enable arrow-parens, no-confusing-arrow */ + const yearDates = yearGroups[year]; + const lastDate = yearDates.reduce((latest, current) => (current > latest ? current : latest)); + lastDateOfYear.push(lastDate); + }); + lastDateOfYear.sort((a, b) => new Date(b) - new Date(a)); + + function buildRepoAggQuery({ calcDates, repoMode }) { + // repoMode: 'not_ja' | 'ja' + const baseBool = { + filter: [ + { terms: { calc_date: lastDateOfYear } }, + { term: { data_type: 'archives.dynamique-ouverture.get-data' } }, + ], + }; + + if (repoMode === 'not_ja') { + baseBool.must_not = [{ term: { repository: 'ja-repository' } }]; + } else if (repoMode === 'ja') { + baseBool.filter.push({ term: { repository: 'ja-repository' } }); + } + + return { + size: 0, + query: { bool: baseBool }, + aggs: { + by_calc_date: { + terms: { field: 'calc_date', size: 1000, order: { _key: 'desc' } }, + aggs: { + nested_data: { + nested: { path: 'data' }, + aggs: { + total_per_year: { + terms: { field: 'data.publication_year', size: 1000 }, + aggs: { + total_sum: { sum: { field: 'data.total' } }, + oa_sum: { sum: { field: 'data.oa' } }, + }, + }, + }, + }, + }, + }, + }, + }; + } + const notJaRes = await Axios.post( + ES_API_URL, + buildRepoAggQuery({ lastDateOfYear, repoMode: 'not_ja' }), + ); + + const jaRes = await Axios.post( + ES_API_URL, + buildRepoAggQuery({ lastDateOfYear, repoMode: 'ja' }), + ); + + // データ整形 + const pickLatestBucket = (res) => res?.data?.aggregations?.by_calc_date?.buckets?.[0]; - const comments = { - observationYear: getObservationLabel(lastObservationSnap, intl), - publicationYear: getObservationLabel(beforeLastObservationSnap, intl), - valueHAL, - valueNotHAL, - }; - - return { - comments, - dataGraph2, - publicationYears, - }; - } - - useEffect(() => { - async function getData() { - try { - const tempData = await GetData(); - setData(tempData); - } catch (e) { - // eslint-disable-next-line no-console - console.error(e); - setError(true); - } finally { - setLoading(false); - } + const toYearOaMap = (bucket) => { + const m = new Map(); + const ys = bucket?.nested_data?.total_per_year?.buckets ?? []; + ys.forEach((b) => m.set(Number(b.key), b.oa_sum?.value ?? 0)); + return m; + }; + + const buildPoints = ({ years, absMap, denomMap }) => years.map((year) => { + const abs = absMap.get(year) ?? 0; + const tot = denomMap.get(year) ?? 0; + return { + x: year, + publicationDate: year, + bsoDomain, + y: tot > 0 ? (abs * 100) / tot : 0, + y_abs: abs, + y_tot: tot, + }; + }); + + // --- jaRes/notJaRes から作る --- + const jaBucket = pickLatestBucket(jaRes); + const notJaBucket = pickLatestBucket(notJaRes); + + const jaOaMap = toYearOaMap(jaBucket); + const notJaOaMap = toYearOaMap(notJaBucket); + + // 年の全集合を作る + const snapYear = parseInt(lastObservationSnap.substring(0, 4), 10); + + const years = Array.from(new Set([...jaOaMap.keys(), ...notJaOaMap.keys()])) + .filter((y) => y > 2012 && y < snapYear) + .sort((a, b) => a - b); + + // 分母（全OA）Map を作る：denom(year) = jaOA + notJaOA + const denomMap = new Map( + years.map((y) => [y, (jaOaMap.get(y) ?? 0) + (notJaOaMap.get(y) ?? 0)]), + ); + + // hal / notHal の data 配列 + const hal = buildPoints({ years, absMap: jaOaMap, denomMap }); + const notHal = buildPoints({ years, absMap: notJaOaMap, denomMap }); + + const dataGraph2 = [ + { + name: capitalize( + intl.formatMessage({ + id: 'app.health-publi.repositories.dynamique-hal.notHal', + }), + ), + data: notHal, + color: getCSSValue('--green-medium-150'), + }, + { + name: capitalize( + intl.formatMessage({ + id: 'app.health-publi.repositories.dynamique-hal.hal', + }), + ), + data: hal, + color: getCSSValue('--acces-ouvert'), + }, + ]; + const publicationYears = years; + return { publicationYears, dataGraph2 }; } - getData(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [beforeLastObservationSnap, lastObservationSnap]); - return { data, isError, isLoading }; + useEffect(() => { + let cancelled = false; + (async () => { + try { + setLoading(true); + setError(false); + const shaped = await getDataByObservationSnaps(); + if (!cancelled) setData(shaped); + } catch (e) { + if (!cancelled) setError(true); + } finally { + if (!cancelled) setLoading(false); + } + })(); + return () => { + cancelled = true; + }; + }, [beforeLastObservationSnap, lastObservationSnap, domain]); + return { data, isLoading, isError }; } -export default useGetData; +export default useGetDataTest; diff --git a/src/components/DataCardsSection/index.js b/src/components/DataCardsSection/index.js index 1b4e3fbb4..20a560028 100644 --- a/src/components/DataCardsSection/index.js +++ b/src/components/DataCardsSection/index.js @@ -248,11 +248,10 @@ export default function DataCardSection({ domain, lang }) { aggregations.by_is_oa.buckets[0].doc_count + aggregations.by_is_oa.buckets[1].doc_count, ); + const hal = aggregations.by_repositories.buckets?.find((item) => item.key === 'HAL'); setTotalHostedDocuments( formatNumberByLang( - aggregations.by_oa_colors.buckets?.find( - (item) => item.key === 'green', - )?.doc_count || 0, + hal?.total, lang, ), ); diff --git a/src/components/Footer/index.js b/src/components/Footer/index.js index 7ea5d813c..1378be229 100644 --- a/src/components/Footer/index.js +++ b/src/components/Footer/index.js @@ -62,6 +62,34 @@ export default function Footer() { + + + + {intl.formatMessage({ id: 'app.footer.terms' })} + + + + + + + {intl.formatMessage({ id: 'app.footer.privacypolicy' })} + + +
diff --git a/src/translations/en.json b/src/translations/en.json index b80420676..5afab1321 100644 --- a/src/translations/en.json +++ b/src/translations/en.json @@ -214,6 +214,8 @@ "app.footer.links": "Links", "app.footer.pnso.text": "Second French Plan for Open Science", "app.footer.pnso.url": "https://www.enseignementsup-recherche.gouv.fr/sites/default/files/2021-10/2e-plan-national-pour-la-science-ouverte-version-anglaise--13715.pdf", + "app.footer.privacypolicy": "Privacy Policy", + "app.footer.privacypolicy.url": "/documents/privacypolicy_en.pdf", "app.footer.project.text": "The project and its team", "app.footer.project.url": "/project-and-team", "app.footer.scanr.text": "scanR, search french research and innovation ecosystem", @@ -221,6 +223,8 @@ "app.footer.scanr.url": "https://data.esr.gouv.fr/EN/", "app.footer.see-also": "Discover also", "app.footer.sources": "Our datasources", + "app.footer.terms": "Terms of Use", + "app.footer.terms.url": "/documents/terms_en.pdf", "app.footer.works-magnet": "Works-magnet, retrieve the scholarly works of your institution", "app.glossary": "Glossary", "app.glossary.acces-ouvert": "Open access (publications)", diff --git a/src/translations/ja.json b/src/translations/ja.json index 0bd2f9809..ad11db40c 100644 --- a/src/translations/ja.json +++ b/src/translations/ja.json @@ -214,6 +214,8 @@ "app.footer.links": "リンク", "app.footer.pnso.text": "（不使用）フランスの第二回オープンサイエンス計画", "app.footer.pnso.url": "（不使用）URLはそのまま出力します。\n\nhttps://www.enseignementsup-recherche.gouv.fr/sites/default/files/2021-10/2e-plan-national-pour-la-science-ouverte-version-anglaise--13715.pdf", + "app.footer.privacypolicy": "プライバシーポリシー", + "app.footer.privacypolicy.url": "/documents/privacypolicy_ja.pdf", "app.footer.project.text": "プロジェクトとチーム", "app.footer.project.url": "（不使用）/プロジェクトとチーム", "app.footer.scanr.text": "（不使用）scanR、フランスの研究と革新のエコシステムを検索する", @@ -221,6 +223,8 @@ "app.footer.scanr.url": "（不使用）https://data.esr.gouv.fr/EN/", "app.footer.see-also": "See Also", "app.footer.sources": "データソース", + "app.footer.terms": "利用規定", + "app.footer.terms.url": "/documents/terms_ja.pdf", "app.footer.works-magnet": "（不使用）Works-magnet、あなたの機関の学術作品を取得します。", "app.glossary": "用語集", "app.glossary.acces-ouvert": "オープンアクセス（OA）", diff --git a/src/utils/Hooks/useFetch.js b/src/utils/Hooks/useFetch.js index da4db23ab..d838968b0 100644 --- a/src/utils/Hooks/useFetch.js +++ b/src/utils/Hooks/useFetch.js @@ -218,7 +218,7 @@ export default function useFetch({ method, options, url }) { 'general.dynamique-ouverture.get-data', 'editeurs.type-ouverture.get-data', 'general.genres-ouverture.get-data-proportion', - 'archives.dynamique-hal.get-data', + 'archives.dynamique-ouverture.get-data', ], }, }, @@ -285,12 +285,35 @@ export default function useFetch({ method, options, url }) { }; res.aggregations.by_genre.buckets.push(genre_buckets_book); } - if (hit._source.data_type === 'archives.dynamique-hal.get-data') { - const repo_data = hit._source.data.find(item => item.publication_year === targetYear); + if (hit._source.data_type === 'archives.dynamique-ouverture.get-data') { + // 対象年のレコードを抽出 + const row = Array.isArray(hit._source.data) + ? hit._source.data.find((it) => it.publication_year === targetYear) + : undefined; + + const oa = Number(row?.oa ?? 0); + const { buckets } = res.aggregations.by_repositories; + + // 既存 HAL バケットの有無を確認 + const existingIdx = buckets.findIndex((b) => b.key === 'HAL'); + const prev = existingIdx >= 0 ? buckets[existingIdx] : { key: 'HAL', doc_count: 0, total: 0 }; + + // 合計更新 + // - total: 全リポジトリ OA 合計 + // - doc_count: ja-repository の OA 合計 + const { repository: repoName = '' } = hit._source; + const nextDocCount = prev.doc_count + (repoName === 'ja-repository' ? oa : 0); + const nextTotal = prev.total + oa; + + // 集計結果を repo_buckets に格納 const repo_buckets = { key: 'HAL', - doc_count: repo_data.jp_repo, + doc_count: nextDocCount, + total: nextTotal, }; + if (existingIdx >= 0) { + buckets.splice(existingIdx, 1); + } res.aggregations.by_repositories.buckets.push(repo_buckets); } });