Beta v.1.0
This commit is contained in:
@@ -0,0 +1,614 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "09f9237a",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Requirement already satisfied: datasets in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (4.4.2)\n",
|
||||
"Requirement already satisfied: tqdm in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (4.67.1)\n",
|
||||
"Requirement already satisfied: pillow in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (12.1.0)\n",
|
||||
"Requirement already satisfied: requests in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (2.32.5)\n",
|
||||
"Requirement already satisfied: filelock in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from datasets) (3.20.3)\n",
|
||||
"Requirement already satisfied: numpy>=1.17 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from datasets) (2.4.1)\n",
|
||||
"Requirement already satisfied: pyarrow>=21.0.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from datasets) (22.0.0)\n",
|
||||
"Requirement already satisfied: dill<0.4.1,>=0.3.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from datasets) (0.4.0)\n",
|
||||
"Requirement already satisfied: pandas in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from datasets) (2.3.3)\n",
|
||||
"Requirement already satisfied: httpx<1.0.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from datasets) (0.28.1)\n",
|
||||
"Requirement already satisfied: xxhash in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from datasets) (3.6.0)\n",
|
||||
"Requirement already satisfied: multiprocess<0.70.19 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from datasets) (0.70.18)\n",
|
||||
"Requirement already satisfied: fsspec<=2025.10.0,>=2023.1.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (2025.10.0)\n",
|
||||
"Requirement already satisfied: huggingface-hub<2.0,>=0.25.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from datasets) (1.3.1)\n",
|
||||
"Requirement already satisfied: packaging in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from datasets) (25.0)\n",
|
||||
"Requirement already satisfied: pyyaml>=5.1 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from datasets) (6.0.3)\n",
|
||||
"Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (3.13.3)\n",
|
||||
"Requirement already satisfied: anyio in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from httpx<1.0.0->datasets) (4.12.1)\n",
|
||||
"Requirement already satisfied: certifi in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from httpx<1.0.0->datasets) (2026.1.4)\n",
|
||||
"Requirement already satisfied: httpcore==1.* in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from httpx<1.0.0->datasets) (1.0.9)\n",
|
||||
"Requirement already satisfied: idna in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from httpx<1.0.0->datasets) (3.11)\n",
|
||||
"Requirement already satisfied: h11>=0.16 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from httpcore==1.*->httpx<1.0.0->datasets) (0.16.0)\n",
|
||||
"Requirement already satisfied: hf-xet<2.0.0,>=1.2.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from huggingface-hub<2.0,>=0.25.0->datasets) (1.2.0)\n",
|
||||
"Requirement already satisfied: shellingham in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from huggingface-hub<2.0,>=0.25.0->datasets) (1.5.4)\n",
|
||||
"Requirement already satisfied: typer-slim in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from huggingface-hub<2.0,>=0.25.0->datasets) (0.21.1)\n",
|
||||
"Requirement already satisfied: typing-extensions>=4.1.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from huggingface-hub<2.0,>=0.25.0->datasets) (4.15.0)\n",
|
||||
"Requirement already satisfied: charset_normalizer<4,>=2 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from requests) (3.4.4)\n",
|
||||
"Requirement already satisfied: urllib3<3,>=1.21.1 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from requests) (2.6.3)\n",
|
||||
"Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (2.6.1)\n",
|
||||
"Requirement already satisfied: aiosignal>=1.4.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (1.4.0)\n",
|
||||
"Requirement already satisfied: attrs>=17.3.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (25.4.0)\n",
|
||||
"Requirement already satisfied: frozenlist>=1.1.1 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (1.8.0)\n",
|
||||
"Requirement already satisfied: multidict<7.0,>=4.5 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (6.7.0)\n",
|
||||
"Requirement already satisfied: propcache>=0.2.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (0.4.1)\n",
|
||||
"Requirement already satisfied: yarl<2.0,>=1.17.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (1.22.0)\n",
|
||||
"Requirement already satisfied: python-dateutil>=2.8.2 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from pandas->datasets) (2.9.0.post0)\n",
|
||||
"Requirement already satisfied: pytz>=2020.1 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from pandas->datasets) (2025.2)\n",
|
||||
"Requirement already satisfied: tzdata>=2022.7 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from pandas->datasets) (2025.3)\n",
|
||||
"Requirement already satisfied: six>=1.5 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.17.0)\n",
|
||||
"Requirement already satisfied: click>=8.0.0 in /home/zin/projects/Thesis/.venv/lib/python3.11/site-packages (from typer-slim->huggingface-hub<2.0,>=0.25.0->datasets) (8.3.1)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!pip install datasets tqdm pillow requests\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "6f0b2e2c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "95f07577d20642b09f2cda6f0b2cca14",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Resolving data files: 0%| | 0/18 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "868d872a109d49f9966f2f19985e7048",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Resolving data files: 0%| | 0/18 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "06741794289540849ad179c5966dcab8",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading data: 0%| | 0/18 [00:00<?, ?files/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "e47aad5270144913996cb5b226213ab9",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00000-of-00018.parquet: 0%| | 0.00/509M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "30d1492a948245e3b6b58e92218cd760",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00001-of-00018.parquet: 0%| | 0.00/504M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "931823b458cb4696b459e9011537cf1e",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00002-of-00018.parquet: 0%| | 0.00/489M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "846f4245b16d4cc096a43c940590ad11",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00003-of-00018.parquet: 0%| | 0.00/507M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "71df201ff1a24811af67458c3fe3f2f4",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00004-of-00018.parquet: 0%| | 0.00/495M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "404dce6c69fc413dbe4aa84c289a0ab6",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00005-of-00018.parquet: 0%| | 0.00/501M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "e52b0bbbfdd14c599f44f02a48542317",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00006-of-00018.parquet: 0%| | 0.00/510M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "172981d77fc941cfa32c05f5a34bf742",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00007-of-00018.parquet: 0%| | 0.00/497M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "cc9d886ff22f4165bf696c8b4d758931",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00008-of-00018.parquet: 0%| | 0.00/512M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "5f118a9923c64ee2aa2001a1414927a3",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00009-of-00018.parquet: 0%| | 0.00/502M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "db61d8d556dc4574adbd8f916f790fa7",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00010-of-00018.parquet: 0%| | 0.00/507M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "75414190b19c4affbe190f6dd4f7bc4f",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00011-of-00018.parquet: 0%| | 0.00/500M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "172aa22ed0c44a289e0ac68b240c13c4",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00012-of-00018.parquet: 0%| | 0.00/504M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "2baa935ed3524a73883909752cb15907",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00013-of-00018.parquet: 0%| | 0.00/491M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "5e716611b29b44788e0bf2e7ad05be5b",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00014-of-00018.parquet: 0%| | 0.00/502M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "d9c0baac101b449794155392f07b49c3",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00015-of-00018.parquet: 0%| | 0.00/504M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b31cdc7f17ac4ac8a04593e8a01a300a",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00016-of-00018.parquet: 0%| | 0.00/507M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "ed6766f750c54b4194957bfe3db78ed6",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/train-00017-of-00018.parquet: 0%| | 0.00/494M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "5454d2ecded64b82a12823f02a7ab12d",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/val-00000-of-00002.parquet: 0%| | 0.00/282M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "62dd1439e0514c98b0c24cc8f600c57e",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/val-00001-of-00002.parquet: 0%| | 0.00/283M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "3a5b966f79314e069251462bff82395f",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/test-00000-of-00004.parquet: 0%| | 0.00/422M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "422974f938924910a0712b30a9c2bd84",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/test-00001-of-00004.parquet: 0%| | 0.00/430M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "f155a08427094de7ad1a5884e623db2b",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/test-00002-of-00004.parquet: 0%| | 0.00/420M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "a94a4621d19f45f690e0064fee83767b",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"data/test-00003-of-00004.parquet: 0%| | 0.00/422M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "50f55b00a27b4213b573b398e5b0d708",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Generating train split: 0%| | 0/94481 [00:00<?, ? examples/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "8658b8414f604f0ca2fd248a214ad4aa",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Generating val split: 0%| | 0/5905 [00:00<?, ? examples/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "d59b7dea75f84b64bb8b262b43730e51",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Generating test split: 0%| | 0/17716 [00:00<?, ? examples/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "0c5815040f0a4a31903348a8327811a5",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Loading dataset shards: 0%| | 0/18 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"DatasetDict({\n",
|
||||
" train: Dataset({\n",
|
||||
" features: ['image', 'label', 'image_id', 'emotion', 'brightness', 'colorfulness', 'facial_expression', 'human_action', 'scene', 'object'],\n",
|
||||
" num_rows: 94481\n",
|
||||
" })\n",
|
||||
" val: Dataset({\n",
|
||||
" features: ['image', 'label', 'image_id', 'emotion', 'brightness', 'colorfulness', 'facial_expression', 'human_action', 'scene', 'object'],\n",
|
||||
" num_rows: 5905\n",
|
||||
" })\n",
|
||||
" test: Dataset({\n",
|
||||
" features: ['image', 'label', 'image_id', 'emotion', 'brightness', 'colorfulness', 'facial_expression', 'human_action', 'scene', 'object'],\n",
|
||||
" num_rows: 17716\n",
|
||||
" })\n",
|
||||
"})\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from datasets import load_dataset\n",
|
||||
"from pathlib import Path\n",
|
||||
"from PIL import Image\n",
|
||||
"import requests\n",
|
||||
"\n",
|
||||
"# куда сохраняем датасет\n",
|
||||
"DATA_DIR = Path(\"../dataset/EmoSet-118K\")\n",
|
||||
"DATA_DIR.mkdir(exist_ok=True, parents=True)\n",
|
||||
"\n",
|
||||
"# загружаем через Hugging Face\n",
|
||||
"ds = load_dataset(\"Woleek/EmoSet-118K\")\n",
|
||||
"\n",
|
||||
"print(ds)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "052ab073",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from tqdm import tqdm\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"def save_split(split):\n",
|
||||
" split_dir = DATA_DIR / split\n",
|
||||
" img_dir = split_dir / \"images\"\n",
|
||||
" img_dir.mkdir(parents=True, exist_ok=True)\n",
|
||||
"\n",
|
||||
" labels_path = split_dir / \"labels.csv\"\n",
|
||||
"\n",
|
||||
" # перезаписываем labels.csv\n",
|
||||
" with open(labels_path, \"w\") as f:\n",
|
||||
" f.write(\"filename,label\\n\")\n",
|
||||
"\n",
|
||||
" for example in tqdm(ds[split]):\n",
|
||||
" img = example[\"image\"] # уже PIL.Image\n",
|
||||
" label = example[\"emotion\"]\n",
|
||||
" image_id = example[\"image_id\"]\n",
|
||||
"\n",
|
||||
" fname = f\"{image_id}.jpg\"\n",
|
||||
" img.save(img_dir / fname)\n",
|
||||
"\n",
|
||||
" with open(labels_path, \"a\") as f:\n",
|
||||
" f.write(f\"{fname},{label}\\n\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "a74ceedf",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 94481/94481 [18:43<00:00, 84.10it/s] \n",
|
||||
"100%|██████████| 5905/5905 [01:08<00:00, 86.57it/s] \n",
|
||||
"100%|██████████| 17716/17716 [02:57<00:00, 100.01it/s]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"save_split(\"train\")\n",
|
||||
"save_split(\"val\")\n",
|
||||
"save_split(\"test\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "thesis-py3.11",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.7"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
import torch
|
||||
from torch.utils.data import Dataset, DataLoader
|
||||
from torchvision import transforms
|
||||
import os
|
||||
import json
|
||||
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class EmoSet(Dataset):
|
||||
ATTRIBUTES_MULTI_CLASS = [
|
||||
'scene', 'facial_expression', 'human_action', 'brightness', 'colorfulness',
|
||||
]
|
||||
ATTRIBUTES_MULTI_LABEL = [
|
||||
'object'
|
||||
]
|
||||
NUM_CLASSES = {
|
||||
'brightness': 11,
|
||||
'colorfulness': 11,
|
||||
'scene': 254,
|
||||
'object': 409,
|
||||
'facial_expression': 6,
|
||||
'human_action': 264,
|
||||
}
|
||||
|
||||
def __init__(self,
|
||||
data_root,
|
||||
num_emotion_classes,
|
||||
phase,
|
||||
):
|
||||
assert num_emotion_classes in (8, 2)
|
||||
assert phase in ('train', 'val', 'test')
|
||||
self.transforms_dict = self.get_data_transforms()
|
||||
|
||||
self.info = self.get_info(data_root, num_emotion_classes)
|
||||
|
||||
if phase == 'train':
|
||||
self.transform = self.transforms_dict['train']
|
||||
elif phase == 'val':
|
||||
self.transform = self.transforms_dict['val']
|
||||
elif phase == 'test':
|
||||
self.transform = self.transforms_dict['test']
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
data_store = json.load(open(os.path.join(data_root, f'{phase}.json')))
|
||||
self.data_store = [
|
||||
[
|
||||
self.info['emotion']['label2idx'][item[0]],
|
||||
item[1],
|
||||
os.path.join(data_root, item[2]),
|
||||
os.path.join(data_root, item[3])
|
||||
]
|
||||
for item in data_store
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def get_data_transforms(cls):
|
||||
transforms_dict = {
|
||||
'train': transforms.Compose([
|
||||
transforms.RandomResizedCrop(224),
|
||||
transforms.RandomHorizontalFlip(),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
|
||||
]),
|
||||
'val': transforms.Compose([
|
||||
transforms.Resize(224),
|
||||
transforms.CenterCrop(224),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
|
||||
]),
|
||||
'test': transforms.Compose([
|
||||
transforms.Resize(224),
|
||||
transforms.CenterCrop(224),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
|
||||
]),
|
||||
}
|
||||
return transforms_dict
|
||||
|
||||
def get_info(self, data_root, num_emotion_classes):
|
||||
assert num_emotion_classes in (8, 2)
|
||||
info = json.load(open(os.path.join(data_root, 'info.json')))
|
||||
if num_emotion_classes == 8:
|
||||
pass
|
||||
elif num_emotion_classes == 2:
|
||||
emotion_info = {
|
||||
'label2idx': {
|
||||
'amusement': 0,
|
||||
'awe': 0,
|
||||
'contentment': 0,
|
||||
'excitement': 0,
|
||||
'anger': 1,
|
||||
'disgust': 1,
|
||||
'fear': 1,
|
||||
'sadness': 1,
|
||||
},
|
||||
'idx2label': {
|
||||
'0': 'positive',
|
||||
'1': 'negative',
|
||||
}
|
||||
}
|
||||
info['emotion'] = emotion_info
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
return info
|
||||
|
||||
def load_image_by_path(self, path):
|
||||
image = Image.open(path).convert('RGB')
|
||||
image = self.transform(image)
|
||||
return image
|
||||
|
||||
def load_annotation_by_path(self, path):
|
||||
json_data = json.load(open(path))
|
||||
return json_data
|
||||
|
||||
def __getitem__(self, item):
|
||||
emotion_label_idx, image_id, image_path, annotation_path = self.data_store[item]
|
||||
image = self.load_image_by_path(image_path)
|
||||
annotation_data = self.load_annotation_by_path(annotation_path)
|
||||
data = {'image_id': image_id, 'image': image, 'emotion_label_idx': emotion_label_idx}
|
||||
|
||||
for attribute in self.ATTRIBUTES_MULTI_CLASS:
|
||||
# if empty, set to -1, else set to label index
|
||||
attribute_label_idx = -1
|
||||
if attribute in annotation_data:
|
||||
attribute_label_idx = self.info[attribute]['label2idx'][str(annotation_data[attribute])]
|
||||
data.update({f'{attribute}_label_idx': attribute_label_idx})
|
||||
|
||||
for attribute in self.ATTRIBUTES_MULTI_LABEL:
|
||||
# if empty, set to 0, else set to 1
|
||||
assert attribute == 'object'
|
||||
num_classes = self.NUM_CLASSES[attribute]
|
||||
attribute_label_idx = torch.zeros(num_classes)
|
||||
if attribute in annotation_data:
|
||||
for label in annotation_data[attribute]:
|
||||
attribute_label_idx[self.info[attribute]['label2idx'][label]] = 1
|
||||
data.update({f'{attribute}_label_idx': attribute_label_idx})
|
||||
|
||||
return data
|
||||
|
||||
def __len__(self):
|
||||
return len(self.data_store)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
data_root = r'F:\common_file_system\EmoSet\EmoSet_v5_划分train-test-val'
|
||||
num_emotion_classes = 8
|
||||
phase = 'train'
|
||||
|
||||
dataset = EmoSet(
|
||||
data_root=data_root,
|
||||
num_emotion_classes=num_emotion_classes,
|
||||
phase=phase,
|
||||
)
|
||||
|
||||
# print(dataset.info)
|
||||
dataloader = DataLoader(dataset, batch_size = 16, shuffle = True)
|
||||
|
||||
for i, data in enumerate(dataloader):
|
||||
pass
|
||||
# print(data['emotion_label_idx'])
|
||||
# print(data['scene_label_idx'])
|
||||
# print(data['facial_expression_label_idx'])
|
||||
# print(data['human_action_label_idx'])
|
||||
# print(data['brightness_label_idx'])
|
||||
# print(data['colorfulness_label_idx'])
|
||||
# print(data['object_label_idx'])
|
||||
# break
|
||||
|
||||
+40
-98
@@ -2,8 +2,8 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "83693ad7",
|
||||
"execution_count": 5,
|
||||
"id": "b92e0213",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -14,119 +14,61 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "99850a99",
|
||||
"id": "1763c51e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Читаем файл аннотаций: ../dataset/DEAM/DEAM_Annotations/annotations/annotations per each rater/song_level/static_annotations_songs_1_2000.csv\n"
|
||||
"✅ УСПЕХ! База создана: ../../dataset/DEAM/music_db.csv\n",
|
||||
"Всего треков в базе: 1744\n",
|
||||
"Пример данных:\n",
|
||||
" song_id valence arousal\n",
|
||||
"0 2 3.1 3.0\n",
|
||||
"1 3 3.5 3.3\n",
|
||||
"2 4 5.7 5.5\n",
|
||||
"3 5 4.4 5.3\n",
|
||||
"4 7 5.8 6.4\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 1. Ищем файл (поднимаемся из src на уровень выше)\n",
|
||||
"deam_root = Path(\"../dataset/DEAM\")\n",
|
||||
"# Точный путь к оригинальным аннотациям\n",
|
||||
"source_path = Path(\"../../dataset/DEAM/DEAM_Annotations/annotations/annotations averaged per song/song_level/static_annotations_averaged_songs_1_2000.csv\")\n",
|
||||
"# Путь, куда сохраним очищенную базу для движка\n",
|
||||
"output_path = Path(\"../../dataset/DEAM/music_db.csv\")\n",
|
||||
"\n",
|
||||
"# Ищем файл статичных аннотаций. Берем первый попавшийся.\n",
|
||||
"csv_files = list(deam_root.rglob(\"*static_annotations*.csv\"))\n",
|
||||
"if not csv_files:\n",
|
||||
" # Если не нашел static, берем вообще любой csv с аннотациями\n",
|
||||
" csv_files = list(deam_root.rglob(\"*.csv\"))\n",
|
||||
"\n",
|
||||
"if not csv_files:\n",
|
||||
" # Если путь неверный или файлов нет, скрипт сразу скажет об этом и покажет полный путь\n",
|
||||
" raise FileNotFoundError(f\"В папке {deam_root.resolve()} не найдено ни одного CSV файла! Проверьте пути.\")\n",
|
||||
"\n",
|
||||
"anno_path = csv_files[0]\n",
|
||||
"print(f\"Читаем файл аннотаций: {anno_path}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "5fbc493f",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Оригинальные колонки в файле: ['workerID', ' SongId', ' Valence', ' Arousal']\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 2. Загружаем и чистим колонки\n",
|
||||
"df = pd.read_csv(anno_path)\n",
|
||||
"print(\"Оригинальные колонки в файле:\", df.columns.tolist())\n",
|
||||
"\n",
|
||||
"# Сносим пробелы по краям и переводим в нижний регистр\n",
|
||||
"df.columns = [str(c).strip().lower() for c in df.columns]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "1e28fece",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Успешно найдены колонки -> ID: 'workerid', Valence: 'valence', Arousal: 'arousal'\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 3. Умный поиск колонок\n",
|
||||
"# Ищем первую колонку, где есть 'id' или 'song'\n",
|
||||
"song_col = next((c for c in df.columns if 'song' in c or 'id' in c), df.columns[0])\n",
|
||||
"# Ищем valence (желательно mean, но сойдет любой)\n",
|
||||
"v_col = next((c for c in df.columns if 'valence' in c and 'mean' in c), \n",
|
||||
" next((c for c in df.columns if 'valence' in c), None))\n",
|
||||
"# Ищем arousal\n",
|
||||
"a_col = next((c for c in df.columns if 'arousal' in c and 'mean' in c), \n",
|
||||
" next((c for c in df.columns if 'arousal' in c), None))\n",
|
||||
"\n",
|
||||
"if not v_col or not a_col:\n",
|
||||
" raise ValueError(f\"Не смог найти Valence или Arousal! Доступные колонки: {df.columns.tolist()}\")\n",
|
||||
"\n",
|
||||
"print(f\"Успешно найдены колонки -> ID: '{song_col}', Valence: '{v_col}', Arousal: '{a_col}'\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "469f651c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Готово! Музыкальная база сохранена: ../dataset/DEAM/music_db.csv\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 4. Сохраняем результат\n",
|
||||
"clean_df = df[[song_col, v_col, a_col]].copy()\n",
|
||||
"clean_df.columns = ['song_id', 'valence', 'arousal']\n",
|
||||
"\n",
|
||||
"output_path = deam_root / \"music_db.csv\"\n",
|
||||
"clean_df.to_csv(output_path, index=False)\n",
|
||||
"print(f\"Готово! Музыкальная база сохранена: {output_path}\")"
|
||||
"if not source_path.exists():\n",
|
||||
" print(f\"❌ Исходный файл не найден по пути: {source_path}\")\n",
|
||||
"else:\n",
|
||||
" # skipinitialspace=True уберет лишние пробелы в названиях колонок, если они есть\n",
|
||||
" df = pd.read_csv(source_path, skipinitialspace=True)\n",
|
||||
" \n",
|
||||
" # Берем только нужные колонки (по твоему примеру)\n",
|
||||
" clean_df = df[['song_id', 'valence_mean', 'arousal_mean']].copy()\n",
|
||||
" \n",
|
||||
" # Переименовываем для простоты кода в движке\n",
|
||||
" clean_df.columns = ['song_id', 'valence', 'arousal']\n",
|
||||
" \n",
|
||||
" # Приводим ID к целому числу (2, 3, 4...), чтобы искать файлы '2.mp3'\n",
|
||||
" clean_df['song_id'] = clean_df['song_id'].astype(int)\n",
|
||||
" \n",
|
||||
" # Сохраняем финальный файл\n",
|
||||
" clean_df.to_csv(output_path, index=False)\n",
|
||||
" \n",
|
||||
" print(f\"✅ УСПЕХ! База создана: {output_path}\")\n",
|
||||
" print(f\"Всего треков в базе: {len(clean_df)}\")\n",
|
||||
" print(\"Пример данных:\")\n",
|
||||
" print(clean_df.head())"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python (my-python-project)",
|
||||
"display_name": "Python (thesis)",
|
||||
"language": "python",
|
||||
"name": "my-python-project"
|
||||
"name": "thesis"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
|
||||
Reference in New Issue
Block a user