diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..4486d3ab0ccd86eefd99deb2adaa117f66b72cad --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +data/ +.ipynb_checkpoints/ +src/ diff --git a/amplitude.png b/amplitude.png deleted file mode 100644 index 2c86334c30a44a592ad9106747ee1b39d37ad838..0000000000000000000000000000000000000000 Binary files a/amplitude.png and /dev/null differ diff --git a/calibracion.ipynb b/calibracion.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..53083b2bacb19c6d39b4222788da6caac2fe67dc --- /dev/null +++ b/calibracion.ipynb @@ -0,0 +1,476 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a4d202eb-9d92-4857-b2af-aba670f9090b", + "metadata": {}, + "source": [ + "# Análisis de Datos Racimo Tormenta" + ] + }, + { + "cell_type": "markdown", + "id": "52bc4c5f-1baf-4e8f-8a1d-0b0e51fa8695", + "metadata": {}, + "source": [ + "Notebook para el análisis de datos del proyecto racimo tormenta" + ] + }, + { + "cell_type": "markdown", + "id": "d277a53c-cccc-4996-8944-620130575372", + "metadata": {}, + "source": [ + "## Librerias " + ] + }, + { + "cell_type": "markdown", + "id": "e96a3270-365f-4f90-9d5a-d2673f176f11", + "metadata": {}, + "source": [ + "Importar las librerias necesarias para el análisis e interacciones de los datos" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d4d6478-6b92-46a4-a849-7fc4d5a3b119", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pylab as plt\n", + "import scipy\n", + "from scipy import stats\n", + "from scipy.fftpack import fftfreq, irfft, rfft\n", + "import sys\n", + "import os\n", + "from matplotlib import cm\n", + "from matplotlib.colors import ListedColormap, LinearSegmentedColormap\n", + "import math\n", + "import datetime as datetime\n", + "import time\n", + "import matplotlib.dates as md\n", + "\n", + "%matplotlib inline\n", + "sys.getdefaultencoding()" + ] + }, + { + "cell_type": "markdown", + "id": "97659221-fc8b-4e1a-b6d9-2ef11fd5410c", + "metadata": {}, + "source": [ + "## Descargar datos " + ] + }, + { + "cell_type": "markdown", + "id": "50ba171a-6d09-4d7a-90ff-b76ce313b09d", + "metadata": {}, + "source": [ + "Ir a https://dataverse.redclara.net/dataverseuser.xhtml?selectTab=apiTokenTab y copiar el **API Token** para definirlo en el siguiente campo" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89723f93-88a1-4bd4-ac98-76a5d4e75c48", + "metadata": {}, + "outputs": [], + "source": [ + "%env API_TOKEN=42401228-20ba-4de1-a889-6aa8ccd89087" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf02c4eb-bfed-478e-8315-2ca2bfbe6470", + "metadata": {}, + "outputs": [], + "source": [ + "%env SERVER_URL=https://dataverse.redclara.net\n", + "%env PERSISTENT_ID=doi:10.21348/FK2/EIQEXC\n", + "%env VERSION=DRAFT \n", + "!curl -L -O -J -H \"X-Dataverse-key:$API_TOKEN\" $SERVER_URL/api/access/dataset/:persistentId/?persistentId=$PERSISTENT_ID" + ] + }, + { + "cell_type": "markdown", + "id": "52c1b78f-5325-4a21-9933-d25cf8e927ab", + "metadata": {}, + "source": [ + "Descomprimir archivo de datos" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c7d3e5e-31fe-42b7-be08-e9c9f793a7da", + "metadata": {}, + "outputs": [], + "source": [ + "!unzip dataverse_files.zip\n", + "!rm dataverse_files.zip\n", + "!rm MANIFEST.TXT" + ] + }, + { + "cell_type": "markdown", + "id": "960fdf13-2b85-4fca-a1fb-ed0aa016e468", + "metadata": {}, + "source": [ + "## Calibración del detector" + ] + }, + { + "cell_type": "markdown", + "id": "3b8b509b-c69e-4c8d-98e2-6f9d7cc825c3", + "metadata": {}, + "source": [ + "Calibración de las mediciones del detector" + ] + }, + { + "cell_type": "markdown", + "id": "440109c1-3272-4702-9a0c-fd5e2b8e6b1d", + "metadata": {}, + "source": [ + "### Vista preliminar de los datos de calibración" + ] + }, + { + "cell_type": "markdown", + "id": "4ed58223-c091-4de4-b9c7-2cf4b3dd674f", + "metadata": {}, + "source": [ + "Cargar datos en formato *array numpy*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78a31578-66f3-4363-80b4-856aabede298", + "metadata": {}, + "outputs": [], + "source": [ + "data = np.loadtxt('data/Lighting_2021_04_13_00_4.dat', comments='#')" + ] + }, + { + "cell_type": "markdown", + "id": "7c03fdf2-6a4a-4d30-87e3-3b025a43f6c1", + "metadata": {}, + "source": [ + "Describir datos " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6733522b-ab2c-4b49-841a-a45bc4dfd377", + "metadata": {}, + "outputs": [], + "source": [ + "from scipy import stats\n", + "stats.describe(data)" + ] + }, + { + "cell_type": "markdown", + "id": "c754f55c-22ff-46b9-a414-7fa07a4497d6", + "metadata": {}, + "source": [ + "### Amplitud y frecuencia de la señal" + ] + }, + { + "cell_type": "markdown", + "id": "badb8819-ef05-4e8c-867c-8b4a03f554d2", + "metadata": {}, + "source": [ + "Función para graficar amplitud y frecuencia." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05f9f0b6-dd57-4e7e-92ca-4718cf85808e", + "metadata": {}, + "outputs": [], + "source": [ + "def Lightning_Analysis(data, dt, Np):\n", + " \n", + " mean = np.mean(data[:,1])\n", + " sigma = np.std(data[:,1])\n", + " peaks = []\n", + " MTFt = [] # Multiple-termination flash (MTF) relative times\n", + " MTSt = [] # Multiple.termination stroke (MTS) relative times\n", + " MTSv = []\n", + " T_count = 0 # Terminations counter\n", + " MTFc = 1 # MTF counter\n", + " MTSc = 0 # MTS counter\n", + " \n", + " \n", + " N = len(data)\n", + " \n", + " MTSw = 1e-3 # time window for differentiating MTF and MTS events\n", + " \n", + " threshold = mean + 5*sigma # Peak threshold\n", + " \n", + " # Termination identification\n", + " for i in range(N):\n", + " if (data[i,1] > threshold):\n", + " T_count += 1\n", + " peaks.append(i)\n", + " t1 = data[i,0]\n", + " \n", + " if T_count > 1:\n", + " Td = t1 - t0\n", + " if Td > MTSw:\n", + " MTFt.append(Td)\n", + " MTFc += 1\n", + " MTSv.append(MTSc)\n", + " MTSc = 0\n", + " else:\n", + " MTSt.append(Td)\n", + " MTSc += 1\n", + " t0 = t1\n", + " \n", + " print (u'Terminations above 5\\u03C3 = %d\\n' %T_count)\n", + " print (u'Number of strokes = %d\\n' %MTFc)\n", + " \n", + " s = data[:,1]\n", + " \n", + " Y = np.fft.fft(s)\n", + " N = len(Y)/2+1\n", + " fa = 1.0/dt\n", + "\n", + " X = np.linspace(0, fa/2, int(N), endpoint=True)\n", + " sfft = np.abs(Y[:int(N)])\n", + "\n", + " print('Sample Time = %.5f s' % dt)\n", + " print('Frequency = %.2f Hz' % fa)\n", + " \n", + " sfft = np.array(sfft)\n", + " pos = int(np.where(sfft[1:-1] == np.amax(sfft[1:-1]))[0])\n", + " frec_pico = 868.35 # X[pos+1]\n", + "\n", + " print (\"Maximum frequency = %.2f Hz\" %frec_pico)\n", + "\n", + " if T_count >= Np:\n", + " \n", + " # Signal plot\n", + "\n", + " plt.figure(figsize = (16,4))\n", + " plt.subplot(1,2,1)\n", + " plt.plot(data[:,0], data[:,1])\n", + " plt.axhline(threshold, color='red')\n", + " plt.xlabel('Time [s]', fontsize = 20)\n", + " plt.ylabel('Amplitude [ADC]', fontsize = 20)\n", + " plt.savefig(\"amplitude.png\", dpi=150)\n", + "\n", + " # Spectrum plotting\n", + "\n", + " plt.subplot(1,2,2)\n", + " plt.axvline(frec_pico, color='red')\n", + " plt.loglog(X, sfft)\n", + " plt.xlabel('Frequency [Hz]', fontsize = 20)\n", + " plt.axis([1e-1,1e5,1e1,1e7])\n", + " plt.grid()\n", + " plt.show()\n", + "\n", + " return frec_pico, peaks, MTFt, MTSt, T_count, MTFc, MTSv" + ] + }, + { + "cell_type": "markdown", + "id": "4da5b0b2-97c4-455b-b9f9-028e817f3fa9", + "metadata": {}, + "source": [ + "Graficar " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "66982705-0db4-43d7-8bd0-22ec2ae7b907", + "metadata": {}, + "outputs": [], + "source": [ + "dt = 10e-6 # sampling period\n", + "Np = 0. # filter signals per number of peaks above 5 sigma\n", + "\n", + "fp1, peaks1, MTFt, MTSt, pN1, MTFc, MTSc = Lightning_Analysis(data, dt, Np) # Returns maximum peak frequency and peak positions" + ] + }, + { + "cell_type": "markdown", + "id": "eccbba23-922e-430b-816b-41ca7212cae5", + "metadata": {}, + "source": [ + "### Escalar señal y enfocar" + ] + }, + { + "cell_type": "markdown", + "id": "c15cc352-a82b-404d-9a47-09860ae25950", + "metadata": {}, + "source": [ + "Definir parámetros de escalado y foco" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aaea4b8a-0639-479e-8f66-ad59fb179ac8", + "metadata": {}, + "outputs": [], + "source": [ + "R1 = 0\n", + "R2 = 200\n", + "P = 5" + ] + }, + { + "cell_type": "markdown", + "id": "5a59c5cb-19db-4872-9046-6f0471498da0", + "metadata": {}, + "source": [ + "Crear nuevo *array* de datos." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d8a87c1-5ab1-4e9f-b2d8-8d10572ebad8", + "metadata": {}, + "outputs": [], + "source": [ + "newdata = data[R1:R2]*[1,P]" + ] + }, + { + "cell_type": "markdown", + "id": "6abec110-ecf4-44e0-b88c-3a3a94b8b4aa", + "metadata": {}, + "source": [ + "Graficar señal escalada" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53d02f88-4245-4b93-a2c4-7da1cb9c4448", + "metadata": {}, + "outputs": [], + "source": [ + "dt = 10e-6 # sampling period\n", + "Np = 0. # filter signals per number of peaks above 5 sigma\n", + "\n", + "fp1, peaks1, MTFt, MTSt, pN1, MTFc, MTSc = Lightning_Analysis(newdata, dt, Np) # Returns maximum peak frequency and peak positions" + ] + }, + { + "cell_type": "markdown", + "id": "6cd50a1f-6003-4002-a1aa-f98af31ffcab", + "metadata": {}, + "source": [ + "### Publicar nuevos datos en dataverse" + ] + }, + { + "cell_type": "markdown", + "id": "78c305e5-7806-432c-8ba6-f179d1f6d863", + "metadata": {}, + "source": [ + "Conectar a dataverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c365000c-82c3-4972-aa80-ad7d76801b43", + "metadata": {}, + "outputs": [], + "source": [ + "from dataverse import Connection\n", + "\n", + "API_TOKEN = os.environ['API_TOKEN']\n", + "host = 'dataverse.redclara.net' # All clients >4.0 are supported\n", + "# Conexión a repositorio\n", + "connection = Connection(host, API_TOKEN)\n", + "# Selección de dataverse a user (storm para Racimo Tormenta)\n", + "dataverse = connection.get_dataverse('storm')" + ] + }, + { + "cell_type": "markdown", + "id": "2f189374-3ad4-4bfc-a11a-67f2c8a0e6eb", + "metadata": {}, + "source": [ + "Guardar *array* en nuevo archivo" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae9b7fd8-9850-4d7e-ab2e-0397c266416b", + "metadata": {}, + "outputs": [], + "source": [ + "np.savetxt('data/Lighting_2021_04_13_00_4-'+connection.token+'.dat', newdata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0a1f7583-ad98-4aba-a67e-518af2319572", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = dataverse.get_dataset_by_title('Lighting_2021_04_13')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "71796740-f56e-4dfa-8d8e-4fac88d4a762", + "metadata": {}, + "outputs": [], + "source": [ + "dataset.upload_filepath('data/Lighting_2021_04_13_00_4-'+connection.token+'.dat')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1d125e8-8bf1-464b-b012-cc648cc8691d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}