Commit 0a5bdaf8 authored by Leonie1.Wagner's avatar Leonie1.Wagner
Browse files

energy encoding set up

parent 02aee98a
......@@ -34,6 +34,7 @@ dict = SpikeTimit.create_dictionary(file=dict_path)
##
# SETTING PARAMETERS
# Parameters to change to filter the data
measurements_per_word = 1; # how often are the states stored per word
measurements_per_phone = 10; # how often are the states stored per phone
......@@ -49,16 +50,8 @@ nbr_of_pop = -1
words = ["that", "had", "she", "me", "your", "all", "like", "don't", "year", "water", "dark", "rag", "oily", "wash", "ask", "carry", "suit"]
target_dialects = "dr1 dr2 dr3 dr4 dr5 dr6 dr7 dr8"
target_gender = "f m"
#target_dialects = [1 2 3 4 5 6 7 8]
#target_gender = "fm" # "fm" "m"
# in_words(df_words) = !isempty(intersect(Set(df_words),Set(words)))
# in_dialect(df_dialect) = df_dialect ∈ target_dialects
# in_gender(df_gender) = occursin(df_gender, target_gender)
# this is a DataFrameMeta macro
#speaker = @where(train,in_dialect.(:dialect), in_gender.(:gender), in_words.(:words))
#speaker
# Filtering the dataframe
# FILTERING THE DATAFRAME AND SELECTING WORDS
speakers = []
for word in words
speaker = @linq train |> # SET THIS TO TRAIN/TEST ACCORDINGLY
......@@ -79,14 +72,14 @@ end
words = [(py_words...)...]
labels = []
word_labels = []
for i in 1:size(words,1)
push!(labels, words[i].word)
push!(word_labels, words[i].word)
end
labels
word_labels
##
##
words[1].phones[1].db
#words[1].phones[1].db
##
using StatsBase
......@@ -106,27 +99,36 @@ jwords =py2j_words(words)
function rate_coding_word(word::SpikeTimit.Word)
times = []
durations = []
phone_labels = []
encoding = Matrix{Float64}(undef, 20, length(word.phones))
for (n,ph) in enumerate(word.phones)
encoding[:,n] = mean(ph.db, dims=2)[:,1]
push!(times, ph.t0 - word.t0)
push!(durations, ph.t1 - ph.t0)
push!(phone_labels, ph.ph)
end
return times, durations, encoding
return times, durations, phone_labels, encoding, word.duration
end
# VARIABLE FOR WORD PROPERTIES IN JULIA
times = []
durations = []
phone_labels = []
rates = []
word_durations = []
for word in jwords
t, d, r = rate_coding_word(word)
t, d, pl, r, wd = rate_coding_word(word)
push!(times, t)
push!(durations, d)
push!(phone_labels, pl)
push!(rates, r)
push!(word_durations, wd)
end
times[1] # starting times (in s?) of each phone in the first word
durations[1]
rates[1] # the rates to each population (rows) for each phone (columns) in the word
word_start_times = zeros(size(word_durations,1))
word_end_times = copy(word_durations)
# NORMALIZING THE RATES
typeof(rates)
size(rates[2],2)
new_rates = copy(rates)
......@@ -135,36 +137,86 @@ for (i, rate) in enumerate(rates)
new_rates[i][:,ph] = (rate[:,ph]./sum(rate[:,ph])) .*8
end
end
new_rates[1] # the rates to each population (rows) for each phone (columns) in the word, normalized to sum to 8
# repeat
# REPEATING THE WORDS
all_times = repeat(times, repetitions)
all_durations = repeat(durations, repetitions)
all_rates = repeat(new_rates, repetitions)
all_labels = repeat(labels, repetitions)
# shuffle
all_word_labels = repeat(word_labels, repetitions)
all_phone_labels = repeat(phone_labels, repetitions)
all_word_durations = repeat(word_durations, repetitions)
all_word_start_times = repeat(word_start_times, repetitions)
all_word_end_times = repeat(word_end_times, repetitions)
# SHUFFLING THE WORDS
using Random
ind = shuffle(1:size(all_times,1))
all_times = all_times[ind,:]
all_durations = all_durations[ind,:]
all_durations = all_durations[ind,:] # durations of phones
all_rates = all_rates[ind,:]
all_labels = all_labels[ind,:]
# to do:
# stack
all_word_labels = all_word_labels[ind,:]
all_phone_labels = all_phone_labels[ind,:]
all_word_durations = all_word_durations[ind,:]
all_word_start_times = all_word_start_times[ind,:]
all_word_end_times = all_word_end_times[ind,:]
# STACKING THE WORDS IN TIME
# Stacking starting times of phones
all_starting_times = copy(all_times)
global_time = 0.
for (w, time_w) in enumerate(all_times)
all_starting_times[w] = time_w .+ global_time
word_duration = sum(all_durations[w])
global_time += word_duration + silence_time
end
# Stacking starting and end times of words
global_time = 0.
for (label, dd) in zip(labels, durations)
@assert(label.duration == dd)
push!(words_transcripts.intervals,(global_time, global_time+dd))
push!(words_transcripts.sign,label.word)
for ph in label.phones
push!(phones_transcripts.intervals, (global_time+ph.t0, global_time+ph.t1))
push!(phones_transcripts.sign,ph.ph)
end
global_time += dd + silence_time
# make the stim
for w in 1:size(all_word_start_times,1)
all_word_start_times[w] = all_word_start_times[w] .+ global_time
all_word_end_times[w] = all_word_end_times[w] .+ global_time
global_time += all_word_durations[w] + silence_time
end
# FLATTENING ARRAYS FOR STIM
flat_all_times = [(all_starting_times...)...]
flat_all_durations = [(all_durations...)...]
flat_all_rates = all_rates[1]
for i in 2:size(all_rates,1)
flat_all_rates = hcat(flat_all_rates, all_rates[i])
end
flat_all_phone_labels = [(all_phone_labels...)...]
# MAKING STIMULUS MATRIX
input_Npop = 20 # nbr of frequencies
target_pops = []
for j in 1:input_Npop
push!(target_pops, j)
end
target_populations = repeat(target_pops, convert(Int, (size(flat_all_times,1)/20)))
stim = zeros(size(flat_all_times,1),4)
for i in 1:size(flat_all_times,1)
stim[i,1] = target_populations[i] # population number of this phone
stim[i,2] = flat_all_times[i]*1000 # start of interval
stim[i,3] = (flat_all_times[i] + flat_all_durations[i])*1000 # end of interval
stim[i,4] = flat_all_rates[:,i][target_populations[i]]
end
# COMPUTING SAVEPOINTS
# End point for phone savepoints
flat_all_end_times = flat_all_times .+ flat_all_durations
word_savepoints = SpikeTimit.get_savepoints(all_start_times=all_word_start_times, all_end_times = all_word_end_times, n_measure = measurements_per_word, timestep=timestep)
phone_savepoints = SpikeTimit.get_savepoints(all_start_times=flat_all_times, all_end_times = flat_all_end_times, n_measure = measurements_per_phone, timestep=timestep)
# SAVING THE INFORMATION AS H5
# stim, phone labels, word labels, phone asvepoints, word savepoints, input npop, timestep
include("../src/ReadWrite.jl")
# Necessary conversions
all_word_labels = convert(Array{String}, all_word_labels)
ReadWrite.save_energy_encoding(stim, all_word_labels, flat_all_phone_labels, word_savepoints, phone_savepoints, input_Npop, timestep, "C:\\Users\\leoni\\Desktop\\3rd_year_AI\\1_Thesis\\litwin-kumar_model_thesis\\data")
# Plots
......
using Base
module ReadWrite
using Dates
using Printf
using Serialization
using HDF5
using OrderedCollections
using JLD
using DataFrames
include("SpikeTimit.jl")
#Reads the data from Spike TIMIT folder and returns the train and test set and dictionary
function read_data_set()
#Create the path strings leading to folders in the data set
test_path = joinpath(@__DIR__,"Spike TIMIT", "test");
train_path = joinpath(@__DIR__,"Spike TIMIT", "train");
dict_path = joinpath(@__DIR__,"DOC", "TIMITDIC.TXT");
#Create the data sets from the given path
train = SpikeTimit.create_dataset(;dir= train_path)
test = SpikeTimit.create_dataset(;dir= test_path)
dict = SpikeTimit.create_dictionary(file=dict_path)
return train, test, dict
end
#Creates a folder to save all the data from one simulation run. Creates data folder if it does not exist yet
function createfolder(; folder_name="")
if !(isdir("data"))
mkdir("data")
end
#=
The default folder to save to is simulation_run_1 (inside data folder)
If this folder exists it will keep incrementing the integer
at the end of the filename until a non-existing folder is found.
=#
if folder_name == ""
n = 1
folder_name = "data/simulation_run_"
folder = string(folder_name,n)
while (isdir(folder)) # while folder already exists create new foldername
n += 1
folder = string(folder_name,n)
end
else
folder = "data/"*folder_name
end
mkdir(folder)
mkdir(string(folder, "/weights"))
mkdir(string(folder, "/mean_weights"))
mkdir(string(folder, "/ie_weights"))
mkdir(string(folder, "/word_states"))
mkdir(string(folder, "/phone_states"))
return folder
end
# WEIGHTS
# Saves the array of network weights to an HDF5 file
function save_network_weights(W::Array, T::Int, rd::String)
filename = abspath(rd*"/weights/weights_T$T.h5") #absolute path #somehow the location gets weird for this one..
fid = h5open(filename,"w")
fid["weights"] = W
fid["tt"] = T
close(fid)
nothing
end
function read_network_weights(rd::String; cache=true)
files = Vector()
folder = rd*"/weights/"
for file_ in readdir(string(@__DIR__, folder))
if startswith(file_,"weights") && endswith(file_,"h5")
filename = string(@__DIR__, folder*file_)
h5open(filename,"r") do fid
tt = read(fid["tt"])
push!(files,(tt, filename))
end
end
sort!(files,by=x->x[1])
end
if cache
ws = Vector{Tuple{Int,Array}}()
for (tt, file_) in files
h5open(file_,"r") do file_
fid = read(file_)
W = get_weights(fid)
push!(ws,(tt, W))
end
end
return ws
else
# channel = Channel()
ws = Vector{Tuple{Int,String}}()
for (tt, file_) in files
push!(ws,(tt, file_))
end
return ws
end
end
# MEAN WEIGHTS
#Saves the array of the mean population weights to an HDF5 file
function save_network_mean_weights(mean_W::Array, T::Int, rd::String)
#filename = joinpath(@__DIR__, rd*"/weights/mean_weights_T$T.h5")
filename = abspath(rd*"/mean_weights/mean_weights_T$T.h5") #absolute path #absolute path #somehow the location gets weird for this one..
fid = h5open(filename,"w")
fid["weights"] = mean_W
fid["tt"] = T
close(fid)
nothing
end
function read_network_mean_weights(rd::String; cache=true)
files = Vector()
folder = rd*"/mean_weights/"
for file_ in readdir(string(@__DIR__, folder))
if startswith(file_,"mean_weights") && endswith(file_,"h5")
filename = string(@__DIR__, folder*file_)
h5open(filename,"r") do fid
tt = read(fid["tt"])
push!(files,(tt, filename))
end
end
sort!(files,by=x->x[1])
end
if cache
ws = Vector{Tuple{Int,Array}}()
for (tt, file_) in files
h5open(file_,"r") do file_
fid = read(file_)
W = get_weights(fid)
push!(ws,(tt, W))
end
end
return ws
else
# channel = Channel()
ws = Vector{Tuple{Int,String}}()
for (tt, file_) in files
push!(ws,(tt, file_))
end
return ws
end
end
# INHIBITORY TO EXCITATORY WEIGHTS
function save_network_ie_weights(weights::Array, T::Int, rd::String)
filename = abspath(rd*"/ie_weights/ie_weights_T$T.h5") #absolute path #absolute path #somehow the location gets weird for this one..
fid = h5open(filename,"w")
fid["weights"] = weights
fid["tt"] = T
close(fid)
nothing
end
function read_network_ie_weights(rd::String; cache=true)
files = Vector()
folder = rd*"/ie_weights/"
for file_ in readdir(string(@__DIR__, folder))
if startswith(file_,"ie_weights") && endswith(file_,"h5")
filename = string(@__DIR__, folder*file_)
h5open(filename,"r") do fid
tt = read(fid["tt"])
push!(files,(tt, filename))
end
end
sort!(files,by=x->x[1])
end
if cache
ws = Vector{Tuple{Int,Array}}()
for (tt, file_) in files
h5open(file_,"r") do file_
fid = read(file_)
W = get_weights(fid)
push!(ws,(tt, W))
end
end
return ws
else
# channel = Channel()
ws = Vector{Tuple{Int,String}}()
for (tt, file_) in files
push!(ws,(tt, file_))
end
return ws
end
end
# POPMEMBERS
function save_network_popmembers(P::Array, rd::String)
filename = abspath(rd*"/popmembers.h5") #absolute path #somehow the location gets weird for this one..
fid = h5open(filename,"w")
fid["popmembers"] = P
close(fid)
nothing
end
function read_network_popmembers(rd::String)
filename = string(@__DIR__, rd*"/popmembers.h5")
if isfile(filename)
file_ = h5open(filename,"r")
fid = read(file_)
P = fid["popmembers"]
close(file_)
return P
end
nothing
end
# STATES
function save_network_state(currents::Array{Float64,1}, exc_mem::Array{Float64,1}, label::String, label_id::Int, rd::String)
filename = abspath(rd*"/state_$label"*"_ID$label_id.h5") #absolute path #somehow the location gets weird for this one..
fid = h5open(filename,"w")
fid["exc_mem"] = exc_mem
fid["currents"] = currents
fid["label"] = label
fid["label_id"] = label_id
close(fid)
nothing
end
#=
Read all network states and return as vector
@param rd::String, the relative path to the data folder for the particular simulation run
@param tt0::Int, the time from which to start taking the measurements into account.
=#
function read_network_states(rd::String; label_id0::Int = 1)
states = Vector()
rd = string(@__DIR__, rd)
for file_ in readdir(rd)
if startswith(file_,"state") && endswith(file_,"h5")
filename = joinpath(rd,file_)
h5open(filename,"r") do fid
if read(fid["label_id"]) >= label_id0
exc_mem = read(fid["exc_mem"])
currents = read(fid["currents"])
label = read(fid["label"])
push!(states, (exc_mem, currents, label))
end
end
end
end
return states
end
# SPIKES
# Save the times at which each neuron spikes (ms), Array shape: (Ncells,Nspikes)
function save_network_spikes(times::Array, rd::String)
filename = abspath(rd*"/spikes.h5")
fid = h5open(filename,"w")
fid["spikes"] = times
close(fid)
nothing
end
#Reads the spikes data saved in an HDF5 file in directory rd
function read_network_spikes(rd::String)
filename = string(@__DIR__, rd*"/spikes.h5")
if isfile(filename)
file_ = h5open(filename,"r")
fid = read(file_)
spikes = fid["spikes"]
close(file_)
return spikes
end
nothing
end
# PARAMETERS
#Saves the network parameters to an HDF5 file. Saves calculated output to a txt file
function save_network_params(dictionary_params::Dict, dictionary_calc::OrderedDict, rd::String)
filename_params = abspath(rd*"/net_params.h5") #absolute path #somehow the location gets weird for this one..
fid = h5open(filename_params,"w")
for (key,value) in dictionary_params
fid[key] = value
end
close(fid)
params_text = ""
for (key, value) in dictionary_calc
params_text = string(params_text, key, " = ", value, "\n") #convert dictionary to nicely formatted list
end
open(string(rd, "/calculated_output.txt"), "w") do io
write(io, params_text) #write all parameters + values to a .txt file
end
nothing
end
#Read the network parameters from an HDF5 file in the given directory rd
function read_network_params(rd::String)
filename = string(@__DIR__, rd*"/net_params.h5")
dict = Dict()
if isfile(filename)
file_ = h5open(filename,"r")
fid = read(file_)
close(file_)
return fid
end
nothing
end
# RATES
# Save the firing rate per population per time-bin (Npop,bins)
function save_network_pop_rates(pop_bin_rates::Array, rd::String)
filename = abspath(rd*"/pop_rates.h5")
fid = h5open(filename,"w")
fid["pop_rates"] = pop_bin_rates
close(fid)
nothing
end
# Save the firing rate per neuron per time-bin (1,bins)
function save_network_neuron_rates(neuron_bin_rates::Array, rd::String; type_of_neuron::String="exc")
filename = abspath(rd*"/"*type_of_neuron*"_neuron_rates.h5")
fid = h5open(filename,"w")
fid["neuron_rates"] = neuron_bin_rates
close(fid)
nothing
end
# Reads the firing rate per population per time-bin (Npop,bins)
function read_network_pop_rates(rd::String)
filename = string(@__DIR__, rd*"/pop_rates.h5")
if isfile(filename)
file_ = h5open(filename,"r")
fid = read(file_)
pop_bin_rates = fid["pop_rates"]
close(file_)
return pop_bin_rates
end
nothing
end
# Reads the firing rate per neuron per time-bin (1,bins)
function read_network_neuron_rates(rd::String; type_of_neuron::String="exc")
filename = string(@__DIR__, rd*"/"*type_of_neuron*"_neuron_rates.h5")
if isfile(filename)
file_ = h5open(filename,"r")
fid = read(file_)
neuron_bin_rates = fid["neuron_rates"]
close(file_)
return neuron_bin_rates
end
nothing
end
# NETWORK (POPMEMBERS AND WEIGHTS)
function save_trained_network(popmembers, weights, rd::String)
filename = abspath(rd*"/trained_network.h5")
fid = h5open(filename,"w")
fid["popmembers"] = popmembers
fid["weights"] = weights
close(fid)
end
function read_trained_network(rd::String)
filename = abspath(rd*"/trained_network.h5")
fid = h5open(filename,"r")
popmembers = read(fid["popmembers"])
weights = read(fid["weights"])
close(fid)
return popmembers, weights
end
function save_input_data(dict::OrderedDict, folder::String)
params_text = ""
#convert dictionary to nicely formatted list
for (key, value) in dict
if typeof(value) == String
params_text = string(params_text, key, " = \"", value, "\"\n")
else
params_text = string(params_text, key, " = ", value, "\n")
end
end
open(string(folder, "/input_data_params.txt"), "w") do io
write(io, params_text) #write all parameters + values to a .txt file
end
end
function save_classification_to_txt(dict::OrderedDict, path::String)
vals_text = ""
for (key, value) in dict
vals_text = string(vals_text, key, " = ", value, "\n") #convert dictionary to nicely formatted list
end
path = joinpath(@__DIR__, path)
open(path, "w") do io
write(io, vals_text) #write all values to a .txt file
end
nothing
end
# STIM MATRIX
function save_run_variables(all_phones, unique_phones, word_labels, phone_labels, rd::String, name::String; word_savepoints = [0], phone_savepoints = [0], target_pops=[0], target_rate=[0])
filename = abspath(rd*"/"*name*"_run_variables.h5")
fid = h5open(filename,"w")
fid["all_phones"] = all_phones
fid["unique_phones"] = unique_phones
fid["word_labels"] = word_labels
fid["phone_labels"] = phone_labels
fid["word_savepoints"] = word_savepoints
fid["phone_savepoints"] = phone_savepoints
fid["target_pops"] = target_pops
fid["target_rate"] = target_rate
close(fid)
end
function read_run_variables(rd::String, name::String)