Commit 0e8b3983 authored by alessio.quaresima's avatar alessio.quaresima
Browse files

language analysis with cross entropy per phone and speaker dialect

parent 91032140
......@@ -35,9 +35,6 @@ dict = SpikeTimit.create_dictionary(file=dict_path)
##
in_words(df_words; target) = !isempty(intersect(Set(df_words),Set(target)))
in_dialect(df_dialect; target) = df_dialect target
in_gender(df_gender; target) = occursin(df_gender, target)
const NYC = 6
const SOUTH = 5
......@@ -82,7 +79,7 @@ function get_all_phones(dataset, words)
end
return all_phones
end
collect(keys(all_phones))
function padding(phone; pad_length=400, pad_zero = -80)
# mat = zeros(20, pad_length)
mat = ones(20, pad_length) .* (-80)
......@@ -125,10 +122,11 @@ function compare_sounds(samples1, samples2)
measures[n,1] = isnothing(ex1) ? 1 : samples1[2][ex1]
measures[n,2] = isnothing(ex2) ? 1 : samples2[2][ex2]
end
return 0.5*(kldivergence(measures[:,1], measures[:,2]) + kldivergence(measures[:,2], measures[:,1]))
# return measures
return jsd(measures[:,1], measures[:,2])
end
jsd(x,y)= 0.5*(kldivergence(x, y) + kldivergence(y,x))
# crossentropy(dr1, dr2)
......@@ -142,55 +140,157 @@ for dr in 1:8
push!(females,@where(train,in_dialect.(:dialect, target=dr), in_gender.(:gender, target=female), in_words.(:words, target=words)) |> x->get_padded_features(get_all_phones(x, words)))
end
# ##
# data = [males..., females...]
# entropy = zeros(16,16)
# for x in 1:16
# for y in 1:16
# entropy[x,y] = compare_sounds(data[x], data[y])
# end
# end
# hclust(entropy)
# hclust(entropy)
# # heatmap(entropy)
##
all_sounds=[]
for (sounds, samples, _) in data
push!(all_sounds, sounds)
end
common_sounds = collect(intersect(Set.(all_sounds)...))
nyc_male_data= get_padded_features(get_all_phones(nyc_male,words))
nyc_female_data= get_padded_features(get_all_phones(nyc_female,words))
south_male_data = get_padded_features(get_all_phones(south_male,words))
south_female_data = get_padded_features(get_all_phones(south_female,words))
##
# measures = compare_sounds(nyc_male_data, south_male_data)
# measures = compare_sounds(nyc_male_data, nyc_female_data)
data = [nyc_male_data, nyc_female_data, south_male_data, south_female_data]
compare_sounds(south_female_data, south_male_data)
compare_sounds(south_female_data, south_female_data)
compare_sounds(nyc_male_data, south_female_data)
south_male_data
south_female_data
nyc_female_data
nyc_male_data
entropy = zeros(4,4)
for x in 1:4
for y in 1:4
entropy[x,y] = compare_sounds(data[x], data[y])
mean_data = zeros(8000, 16, length(common_sounds))
std_data = zeros(8000, 16, length(common_sounds))
for (n,s) in enumerate(common_sounds)
speaker=0
for (sounds, samples, feats) in data
speaker+=1
id = findfirst(x->x==s, sounds)
mean_data[:, speaker, n] = mean(feats[id], dims=2)[:,1]
std_data[:, speaker, n] = std(feats[id], dims=2)[:,1]
end
end
heatmap(entropy, yflip=true)
# heatmap(reshape(std_data[:,16,10],20,400))
# heatmap(reshape(mean_data[:,16,10],20,400))
##
heatmap(map(x->reshape(x, 20,400), mean.(nyc_male_data[3], dims=2))[1])
heatmap(map(x->reshape(x, 20,400), mean.(nyc_male_data[3], dims=2))[2])
heatmap(map(x->reshape(x, 20,400), mean.(nyc_male_data[3], dims=2))[3])
heatmap(map(x->reshape(x, 20,400), mean.(nyc_male_data[3], dims=2))[4])
heatmap(map(x->reshape(x, 20,400), mean.(nyc_male_data[3], dims=2))[5])
heatmap(map(x->reshape(x, 20,400), mean.(nyc_male_data[3], dims=2))[6])
heatmap(map(x->reshape(x, 20,400), mean.(nyc_male_data[3], dims=2))[35])
phone_div =zeros(35,35,2)
for speaker in 1:16
for x in 1:35
for y in 1:35
phone_div[x,y,1]+=jsd(abs.(mean_data[:,speaker,x]),abs.(mean_data[:,speaker,y]))
phone_div[x,y,2]+=jsd(abs.(std_data[:,speaker,x]),abs.(std_data[:,speaker,y]))
end
end
end
speaker_div =zeros(16,16,2)
for phone in 1:35
for spx in 1:16
for spy in 1:16
speaker_div[spx,spy,1]+=jsd(abs.(mean_data[:,spx,phone]),abs.(mean_data[:,spy,phone]))
speaker_div[spx,spy,2]+=jsd(abs.(std_data[:,spx,phone]),abs.(std_data[:,spy,phone]))
end
end
end
##
using Clustering
drs = [
"m_New England",
"m_Northern",
"m_North Midland",
"m_South Midland",
"m_Southern",
"m_New York City",
"m_Western",
"m_Moved around",
"f_New England",
"f_Northern",
"f_North Midland",
"f_South Midland",
"f_Southern",
"f_New York City",
"f_Western",
"f_Moved around"]
drs= repeat(drs,2)
order = hclust(speaker_div[:,:,1],linkage=:complete).order
heatmap(speaker_div[order,order,1])
heatmap(speaker_div[order,order,2])
pdialects = plot!(xticks=(1:16,drs[order]),yticks=(1:16, drs[order]), xrotation=45, colorbar=false, tickfontsize=10, title="Dialects cross-entropy")
##
heatmap(phone_div)
heatmap(reshape(all_data[:,6,10], 20,400))
all_data[:,6,10]
padding(all_phones["t"][1])
order = hclust(phone_div, linkage=:complete).order
heatmap(phone_div[order,order])
common_sounds
pphones =plot!(xticks=(1:35,collect(common_sounds)[order]),yticks=(1:35, common_sounds[order]), xrotation=45, colorbar=false, tickfontsize=10, title="Phones cross-entropy")
##
savefig(pphones, "phones_similarities.png")
savefig(pdialects, "dialects_crossentropy.png")
p = Plots.plot()
for phone in all_phones
if length(phone[2]) > 5
p = histogram!(collect(map(r-> size(r.db)[2], phone[2])), bins=0:20:400, label=phone[1], alpha=0.5)
end
# plot!(title=phone[1])
# push!(plots,p)
##
##
variations =zeros(8000,16,35)
for phone in 1:35
for sp in 1:16
variations[:,sp, phone]= std
end
end
#
data[1][3]
#
# mean(data[1][3][10], dims=2)[:,1]
# # , dims=2
# # , dims=2
#
# data[1][3][10]
# heatmap(map(x->reshape(x, 20,400), mean.(data[1][3][2], dims=2))[1])
# heatmap(map(x->reshape(x, 20,400), mean.(nyc_male_data[3], dims=2))[2])
# heatmap(map(x->reshape(x, 20,400), mean.(nyc_male_data[3], dims=2))[3])
# heatmap(map(x->reshape(x, 20,400), mean.(male_data[3], dims=2))[4])
# heatmap(map(x->reshape(x, 20,400), mean.(nyc_male_data[3], dims=2))[5])
# heatmap(map(x->reshape(x, 20,400), mean.(nyc_male_data[3], dims=2))[6])
# heatmap(map(x->reshape(x, 20,400), mean.(nyc_male_data[3], dims=2))[35])
##
phone_div =ones(16,16,35,35)*-1
for sp1 in 1:16
for sp2 in 1:sp1
for x in 1:35
for y in 1:x
phone_div[sp1,sp2,x,y]=jsd(abs.(all_data[:,sp1,x]),abs.(all_data[:,sp2,y]))
phone_div[sp2,sp1,y,x]= phone_div[sp1,sp2,x,y]
end
end
end
end
##
heatmap(phone_div[1,:,4,:])
hclust(phone_div)
##
word = "little"
matches = SpikeTimit.find_word(;df=train, word=word)
water_f = @where(matches, SpikeTimit.in_gender.(:gender,target="f")) |> x-> TIMIT.get_spectra(x |> Pandas.DataFrame, target_words=word)[1]
water_m = @where(matches, SpikeTimit.in_gender.(:gender,target="m")) |> x-> TIMIT.get_spectra(x |> Pandas.DataFrame, target_words=word)[2]
function stack_phones(word)
phones = []
for ph in word.phones
push!(phones,ph.db)
end
return phones
end
# map(r-> size(r.db)[2],all_phones["ae"])
# Plots.plot(plots...)
# all_phones
p
using Plots
p = Plots.plot(heatmap(hcat(stack_phones(water_f)...), title="Word: "*word),heatmap(hcat(stack_phones(water_m)...), ylabel="Frequency", xlabel="Duration (ms)"), layout=(2,1), colorbar=false, guidefontsize=18, tickfontsize=15, titlefontsize=20)
savefig(p,"word_little.png")
q = Plots.plot(heatmap(water_f.phones[4].db, title="Phone: EL"),heatmap(water_m.phones[4].db, ylabel="Frequency", xlabel="Duration (ms)"), layout=(2,1), colorbar=false, guidefontsize=18, tickfontsize=15, titlefontsize=20)
savefig(q,"phone_little.png")
##
heatmap(padding(all_phones["ae"][1]))
heatmap(padding(all_phones["ae"][2]))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment