Skip to content

J
juobs
Commit 8aa2abdb authored by Antonino D'Anna's avatar Antonino D'Anna
Browse files
Options

changes in fit_routines plus tests

parent bf1b33cf
Hide whitespace changes
Inline Side-by-side
Showing with 230 additions and 738 deletions
dev/read_meson_vec_gamma.jl deleted 100644 → 0
View file @ bf1b33cf
using juobs
function dev_read_mesons(path::String, gamma::Vector{Tuple{String,String}}; id::Union{String, Nothing}=nothing, legacy::Bool=false,
nnoise_trunc::Union{Int64, Nothing}=nothing)
type = Vector{Tuple{Int64,Int64}}(undef,length(gamma))
for i in eachindex(gamma)
G1,G2 = gamma[i]
type[i] = (findfirst(x-> x==G1, juobs.gamma_name) - 1, findfirst(x-> x==G2, juobs.gamma_name) - 1)
end
if isnothing(id)
bname = basename(path)
m = findfirst(r"[A-Z][0-9]{3}r[0-9]{3}", bname)
id = bname[m[1:4]]
end
data = open(path, "r")
g_header = juobs.read_GHeader(path)
c_header = juobs.read_CHeader(path, legacy=legacy)
ncorr = g_header.ncorr
tvals = g_header.tvals
nnoise = g_header.nnoise
nnoise_trunc = isnothing(nnoise_trunc) ? nnoise : min(nnoise, nnoise_trunc)
fsize = filesize(path)
datsize = 4 + sum(getfield.(c_header, :dsize)) * tvals * nnoise #data_size / ncnfg
ncfg = div(fsize - g_header.hsize - sum(getfield.(c_header, :hsize)), datsize) #(total size - header_size) / data_size
corr_match = findall(x-> (x.type1,x.type2) in type, c_header)
seek(data, g_header.hsize + sum(c.hsize for c in c_header))
res = Array{juobs.CData}(undef, length(corr_match))
data_re = Array{Float64}(undef, length(corr_match), ncfg, tvals)
data_im = zeros(length(corr_match), ncfg, tvals)
vcfg = Array{Int32}(undef, ncfg)
for icfg = 1:ncfg
vcfg[icfg] = read(data, Int32)
c=1
for k = 1:ncorr
if k in corr_match
if c_header[k].is_real == 1
tmp = Array{Float64}(undef, tvals*nnoise)
read!(data, tmp)
tmp2 = reshape(tmp, (nnoise, tvals))
tmp2 = mean(tmp2[1:nnoise_trunc, :], dims=1)
data_re[c, icfg, :] = tmp2[1, :]
elseif c_header[k].is_real == 0
tmp = Array{Float64}(undef, 2*tvals*nnoise)
read!(data, tmp)
tmp2 = reshape(tmp, (2, nnoise, tvals))
tmp2 = mean(tmp2[:, 1:nnoise_trunc, :], dims=2)
data_re[c, icfg, :] = tmp2[1, 1, :]
data_im[c, icfg, :] = tmp2[2, 1, :]
end
c += 1
else
seek(data, position(data) + c_header[k].dsize*tvals*nnoise)
end
end
end
for c in eachindex(corr_match)
res[c] = juobs.CData(c_header[corr_match[c]], vcfg, data_re[c, :, :], data_im[c, :, :], id)
end
close(data)
return res
end
function dev_read_mesons(path::Vector{String}, gamma::Vector{Tuple{String,String}}; id::Union{String, Nothing}=nothing, legacy::Bool=false,
nnoise_trunc::Union{Int64, Nothing}=nothing)
res = [dev_read_mesons(p, gamma, id=id, legacy=legacy, nnoise_trunc=nnoise_trunc) for p in path]
nrep = length(res)
ncorr = length(res[1])
cdata = Vector{Vector{juobs.CData}}(undef, ncorr)
for icorr = 1:ncorr
cdata[icorr] = Vector{juobs.CData}(undef, nrep)
for r = 1:nrep
cdata[icorr][r] = res[r][icorr]
end
end
return cdata
end
function dev_read_mesons_correction(path::String, gamma::Vector{Tuple{String,String}}; id::Union{String, Nothing}=nothing, legacy::Bool=false,
nnoise_trunc::Union{Int64, Nothing}=nothing)
type = Vector{Tuple{Int64,Int64}}(undef,length(gamma))
for i in eachindex(gamma)
G1,G2 = gamma[i]
type[i] = (findfirst(x-> x==G1, juobs.gamma_name) - 1, findfirst(x-> x==G2, juobs.gamma_name) - 1)
end
if isnothing(id)
bname = basename(path)
m = findfirst(r"[A-Z][0-9]{3}r[0-9]{3}", bname)
id = bname[m[1:4]]
#id = parse(Int64, bname[m[2:4]])
end
data = open(path, "r")
g_header = juobs.read_GHeader(path)
c_header = juobs.read_CHeader(path, legacy=legacy)
ncorr = g_header.ncorr
tvals = g_header.tvals
nnoise = g_header.nnoise
nnoise_trunc = isnothing(nnoise_trunc) ? nnoise : min(nnoise, nnoise_trunc)
fsize = filesize(path)
datsize = 4 + sum(getfield.(c_header, :dsize)) * tvals * nnoise #data_size / ncnfg
ncfg = div(fsize - g_header.hsize - sum(getfield.(c_header, :hsize)), datsize) #(total size - header_size) / data_size
corr_match = findall(x-> (x.type1,x.type2) in type, c_header)
seek(data, g_header.hsize + sum(c.hsize for c in c_header))
res = Array{juobs.CData}(undef, div(length(corr_match), 2)) # Modification: total length is divided by 2
data_re = zeros(div(length(corr_match), 2), ncfg, tvals) # Modification: total length is divided by 2
data_im = zeros(div(length(corr_match), 2), ncfg, tvals) # Modification: total length is divided by 2
vcfg = Array{Int32}(undef, ncfg)
for icfg = 1:ncfg
vcfg[icfg] = read(data, Int32)
c = 1
sgn = +1 # sign it changes at ncorr / 2. O_exact - O_sloppy
for k = 1:ncorr
if k in corr_match
if c_header[k].is_real == 1
tmp = Array{Float64}(undef, tvals*nnoise)
read!(data, tmp)
tmp2 = reshape(tmp, (nnoise, tvals))
tmp2 = mean(tmp2[1:nnoise_trunc, :], dims=1)
data_re[c, icfg, :] = data_re[c, icfg, :] + sgn * tmp2[1, :]
elseif c_header[k].is_real == 0
tmp = Array{Float64}(undef, 2*tvals*nnoise)
read!(data, tmp)
tmp2 = reshape(tmp, (2, nnoise, tvals))
tmp2 = mean(tmp2[:, 1:nnoise_trunc, :], dims=2)
data_re[c, icfg, :] = data_re[c, icfg, :] + sgn * tmp2[1, 1, :]
data_im[c, icfg, :] = data_im[c, icfg, :] + sgn * tmp2[2, 1, :]
end
c += 1
else
seek(data, position(data) + c_header[k].dsize*tvals*nnoise)
end
if k == div(ncorr, 2)
c = 1
sgn = -1
end
end
end
for c = 1:div(length(corr_match), 2)
res[c] = juobs.CData(c_header[corr_match[c]], vcfg, data_re[c, :, :], data_im[c, :, :], id)
end
close(data)
return res
end
function dev_read_mesons_correction(path::Vector{String}, gamma::Vector{Tuple{String,String}}; id::Union{String, Nothing}=nothing, legacy::Bool=false,
nnoise_trunc::Union{Int64, Nothing}=nothing)
res = [dev_read_mesons_correction(p, gamma, id=id, legacy=legacy, nnoise_trunc=nnoise_trunc) for p in path]
nrep = length(res)
ncorr = length(res[1])
cdata = Vector{Vector{juobs.CData}}(undef, ncorr)
for icorr = 1:ncorr
cdata[icorr] = Vector{juobs.CData}(undef, nrep)
for r = 1:nrep
cdata[icorr][r] = res[r][icorr]
end
end
return cdata
end
\ No newline at end of file
src/juobs_tools.jl
View file @ 8aa2abdb
......@@ -685,10 +685,10 @@ function plat_av(obs::Vector{uwreal}, plat::Vector{Int64}, wpm::Union{Dict{Int64
end
plat_av(obs::Vector{uwreal}, plat::Vector{Int64}, W::Vector{Float64},wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing) = sum(W .* obs[plat[1]:plat[2]]) / sum(W)
function plat_av(obs::Vector{uwreal}, plat::Vector{Int64}, W::Matrix{Float64},wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing)
o = obs[plat[1]:plat[2]]
return sum([o[i]*W[i,j]*o[j] for i in axes(W,1), j in axes(W,2)])/sum([o[i]*W[i,j] for i in axes(W,1), j in axes(W,2)])
end
function model_av(fun::Function, y::Vector{uwreal}, guess::Float64;
tm::Vector{Int64}, tM::Vector{Int64}, k::Int64,
wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing)
......@@ -823,11 +823,10 @@ function bayesian_av(fun::Function, y::Array{uwreal}, tmin_array::Array{Int64},
for j in tmax_array ## vary tmax
try
x = [i for i in INDEX+1:1:j]
yy = y[INDEX+1:1:j]
Ncut = total - length(x)
yy = y[INDEX+1:1:j]
Ncut = total - length(x)
dy = err.(yy)
W = 1 ./ dy .^2
p00 = [0.5 for i in 1:1:k]
chisq = gen_chisq(fun,x,dy)
fit = curve_fit(fun,x,value.(yy),W,p00)
......@@ -835,20 +834,13 @@ function bayesian_av(fun::Function, y::Array{uwreal}, tmin_array::Array{Int64},
isnothing(wpm) ? uwerr(up[1]) : uwerr(up[1],wpm)
chi2 = sum(fit.resid.^2) # * dof(fit) / chi_exp
Q = pvalue(chisq, sum(fit.resid.^2), value.(up), yy, wpm=wpm, W=W, nmc=10000)
# if chi2 / dof(fit) > 5. || abs(value(up[1])) > 2. || err(up[1])/value(up[1])*100 >= 10.
# error()
# end
# println(up[1], " ", up[2], " ", up[3])
push!(Pval, Q)
# push!(AIC, chi2 + 2*k + 2*Ncut) # AIC criteria
push!(AIC, chi2 - 2*chi_exp) # TIC criteria
push!(chi2chi2exp, chi2 / dof(fit)) #substitute chi_exp with dof(fit) for chi^2/dof
push!(p1, up[1])
push!(mods,string("[", INDEX+1, ",", j, "]"))
# println("chi2_vs_exp " , chi2 / dof(fit), " chi2/dof ", sum(fit.resid.^2)/dof(fit))
catch
catch
@warn string(":/ Negative window for error propagation at tmin = ", INDEX, ", tmax = ", j, "; skipping that point")
end
end
......@@ -1248,13 +1240,13 @@ function get_chi(model::Function,x,par,data,W::Vector{Float64})
return sum((data.-model(x,par)).^2 .* W)
end
@doc raw"""
@doc """
fit_routine(model::Function, xdata::Array{<:Real}, ydata::Array{uwreal}, npar::Int64;
wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}}=Dict{Int64,Vector{Float64}}(),
corr::Bool = true,
W::VecOrMat{Float64}=Vector{Float64}(),
guess::Vector{Float64} = fill(0.5,npar),
logfile
logfile,
C::AbstractMatrix{Float64} = Matrix{Float64}());
It executes a fit of the `ydata` with the fit function `model`.
......@@ -1282,561 +1274,226 @@ It returns a NamedTuple with names:
- `:pval`: pvalue
"""
function fit_routine(model::Function,
xdata::AbstractArray{<:Real},
ydata::Vector{uwreal},
npar::Int64;
wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}}=Dict{Int64,Vector{Float64}}(),
corr::Bool = true,
W::AbstractVecOrMat{Float64}=Vector{Float64}(),
guess::AbstractVector{Float64} = fill(0.5,npar),
logfile = nothing,
C::AbstractMatrix{Float64} = Matrix{Float64}(undef,0,0));
nobs = length(ydata)
if length(xdata)!=nobs
error("xdata and ydata must be the of same size")
end
[uwerr(y, wpm) for y in ydata]
if length(W) ==0
if corr
C = length(C) ==0 ? ADerrors.cov(ydata,wpm) : C
W = LinearAlgebra.pinv(C);
W = (W'+W)*0.5
else
W = [1/y.err^2 for y in ydata]
end
end
fit = LsqFit.curve_fit(model,xdata,ADerrors.value.(ydata),W,guess)
chisq(p,d) = get_chi(model,xdata,p,d,W)
chi2 = sum(fit.resid.^2)
par = fit_error(chisq,LsqFit.coef(fit),ydata,wpm,W,false)
# if the fit is correlated we have already C, no need to recompute it.
chiexp,pval = if length(C) !=0
GammaMethod.chiexp(chisq,LsqFit.coef(fit),value.(ydata),C,W),juobs.pvalue(chisq,chi2,LsqFit.coef(fit),ydata,W,C=C)
else
ADerrors.chiexp(chisq,LsqFit.coef(fit),ydata,wpm,W=W),juobs.pvalue(chisq,chi2,LsqFit.coef(fit),ydata,W)
end
if !isnothing(logfile)
uwerr.(par);
println(logfile, "juobs.fit_routine output:")
println(logfile, "\t\tFit routine in [$(xdata[1]),...,$(xdata[end])]")
println(logfile, "\t\tparameters :", par...)
println(logfile, "\t\tchi2: ", chi2)
println(logfile, "\t\tchiexp: ", chiexp)
println(logfile, "\t\tpvalue: ", pval)
end
return (par=par,chi2=chi2,chiexp=chiexp,pval=pval)
end
#= function fit_routine(model::Vector{Function}, xdata::Vector{Array{Float64, N}} where N, ydata::Vector{Array{uwreal, N}} where N, param::Int64; wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing,
correlated_fit::Bool=false)
if !(length(model) == length(xdata) == length(ydata))
error("Dimension mismatch")
end
N = length(model)
dat = ydata[1]
idx = Vector{Vector{Int64}}(undef, N)
e = Vector{Vector{Float64}}(undef, N)
j = 1
for i = 1:N
if isnothing(wpm)
uwerr.(ydata[i])
xdata::AbstractArray{<:Real},
ydata::AbstractArray{uwreal},
npar::Int64;
wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}}=Dict{Int64,Vector{Float64}}(),
corr::Bool = true,
W::AbstractArray{Float64}=Vector{Float64}(),
guess::AbstractVector{Float64} = fill(0.5,npar),
logfile = nothing,
C::AbstractArray{Float64} = Matrix{Float64}(undef,0,0));
nobs = length(ydata)
if length(xdata)!=nobs
error("xdata and ydata must be the of same size")
end
[uwerr(y, wpm) for y in ydata]
if length(W) ==0
if corr
C = length(C) ==0 ? ADerrors.cov(ydata,wpm) : C
W = LinearAlgebra.pinv(C);
W = (W'+W)*0.5
else
# println(yaux for yaux in ydata[i])
for yaux in ydata[i]
[uwerr(yaux[k], wpm) for k in 1:length(yaux)]
end
# [[uwerr(yaux[k], wpm) for k in eachindex(yaux)] for yaux in ydata[i]]
end
e[i] = err.(ydata[i])
if i > 1
dat = vcat(dat, ydata[i])
end
stp = j + length(ydata[i]) - 1
idx[i] = collect(j:stp)
j = stp + 1
end
if !correlated_fit
chisq = (par, dat) -> sum([sum((dat[idx[i]] .- model[i](xdata[i], par)).^2 ./e[i].^2) for i=1:N])
else
inv_cov_tot = juobs.inv_covar(vcat(ydata...)) #[inv_covar_multi_id(dat[idx[ii]]) for ii in 1:N]
function chisq(par, dat)
res = Vector{uwreal}(undef,0)
for ii in 1:N
res = vcat(res, dat[idx[ii]] .- model[ii](xdata[ii],par))
end
chi = reshape(res, 1,:) * inv_cov_tot*res
return chi[1]
W = [1/y.err^2 for y in ydata]
end
end
min_fun(t) = chisq(t, value.(dat))
p = fill(0.5, param)
#println(chisq(p,dat))
sol = optimize(min_fun, p, LBFGS())
#println(chisq(sol.minimizer,dat))
if !correlated_fit
@info("Uncorrelated fit ")
(upar, chi2_exp) = isnothing(wpm) ? fit_error(chisq, sol.minimizer, dat) : fit_error(chisq, sol.minimizer, dat, wpm)
else
@info("Correlated fit ")
#inv_cov_merge = zeros(size(inv_cov_tot[1],1)*N, size(inv_cov_tot[1],1)*N )
#count = 1
#for k in 1:N
# dim_tmp = size(inv_cov_tot[k], 1)
# inv_cov_merge[count:dim_tmp+count-1, count:dim_tmp+count-1] = inv_cov_tot[k]
# count +=dim_tmp
#end
(upar, chi2_exp) = isnothing(wpm) ? fit_error(chisq, sol.minimizer, dat, W=inv_cov_tot) : fit_error(chisq, sol.minimizer, dat, wpm, W=inv_cov_tot)
end
println("Chisq / chiexp: ", min_fun(sol.minimizer), " / ", chi2_exp, " (dof: ", length(dat) - param,")")
chis2_corrected = (length(dat) - param) * min_fun(sol.minimizer) / chi2_exp
println("Chisq corrected: ", chis2_corrected)
@info("Returns: params, chi2, chi2_exp\n")
return upar, min_fun(sol.minimizer) , chi2_exp
end
function fit_routine(model::Function, xdata::Array{uwreal}, ydata::Array{uwreal}, param::Int64=3;
wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing, covar::Bool=false)
Nalpha = size(xdata, 2) # number of x-variables
Ndata = size(xdata, 1) # number of datapoints
if isnothing(wpm)
uwerr.(ydata)
uwerr.(xdata)
else
[uwerr(yaux, wpm) for yaux in ydata]
[uwerr(xaux, wpm) for xaux in xdata]
end
yval = value.(ydata)
yer = err.(ydata)
xval = value.(xdata)
xer = err.(xdata)
dat = Vector{Float64}(undef, Ndata * (Nalpha+1))
ddat = Vector{Float64}(undef, Ndata * (Nalpha+1))
data = Vector{uwreal}(undef, Ndata * (Nalpha+1))
for i = 1:Nalpha
dat[(i-1)*Ndata+1:i*Ndata] = xval[:, i]
ddat[(i-1)*Ndata+1:i*Ndata] = xer[:, i]
data[(i-1)*Ndata+1:i*Ndata] = xdata[:, i]
end
dat[Nalpha*Ndata+1:end] = yval
ddat[Nalpha*Ndata+1:end] = yer
data[Nalpha*Ndata+1:end] = ydata
# Guess
fit = curve_fit(model, xval, yval, 1.0 ./ yer.^2, fill(0.5, param))
# Generate chi2 + solver
if covar
aux = Vector{Vector{uwreal}}(undef, Ndata)
for k = 1:Ndata
aux[k] = Vector{uwreal}(undef, Nalpha+1)
for i = 1:Nalpha
aux[k][i] = xdata[k, i]
end
aux[k][Nalpha+1] = ydata[k]
end
C = isnothing(wpm) ? [ADerrors.cov(aux[k]) for k = 1:Ndata] : [ADerrors.cov(aux[k], wpm) for k = 1:Ndata]
chisq_full_cov(p, d) = get_chi2_cov(model, d, C, p, Nalpha)
min_fun_cov(t) = chisq_full_cov(t, dat)
sol = optimize(min_fun_cov, vcat(fit.param, dat[1:Nalpha*Ndata]), method=LBFGS())
(upar, chi2_exp) = isnothing(wpm) ? fit_error(chisq_full_cov, sol.minimizer, data) : fit_error(chisq_full_cov, sol.minimizer, data, wpm)
println("Chisq / chiexp: ", min_fun_cov(sol.minimizer), " / ", chi2_exp, " (dof: ", length(ydata) - param,")")
chis2_corrected = (length(ydata) - param) * min_fun_cov(sol.minimizer) / chi2_exp
println("Chisq corrected: ", chis2_corrected)
return upar, min_fun_cov(sol.minimizer) / chi2_exp, chi2_exp
fit = LsqFit.curve_fit(model,xdata,ADerrors.value.(ydata),W,guess)
else
chisq_full(p, d) = get_chi2(model, d, ddat, p, Nalpha)
min_fun(t) = chisq_full(t, dat)
sol = optimize(min_fun, vcat(fit.param, dat[1:Nalpha*Ndata]), method=LBFGS())
(upar, chi2_exp) = isnothing(wpm) ? fit_error(chisq_full, sol.minimizer, data) : fit_error(chisq_full, sol.minimizer, data, wpm)
println("Chisq / chiexp: ", min_fun(sol.minimizer), " / ", chi2_exp, " (dof: ", length(ydata) - param,")")
chis2_corrected = (length(ydata) - param) * min_fun(sol.minimizer) / chi2_exp
println("Chisq corrected: ", chis2_corrected)
return upar, min_fun(sol.minimizer) / chi2_exp, chi2_exp
end
#### chisq_full, min_fun out of conditional ->
#### COMPILER WARNING ** incremental compilation may be fatally broken for this module **
# Info
#for i = 1:length(upar)
# isnothing(wpm) ? uwerr(upar[i]) : uwerr(upar[i], wpm)
# print("\n Fit parameter: ", i, ": ")
# details(upar[i])
#end
end
function global_fit_routine(models::Vector{Function}, xdata::Vector{Array{Float64, 1}} , ydata::Vector{Array{uwreal, 1}}, n_par::Int64; guess_param::Vector{Float64}=fill(0.5, n_par), log::Bool=true,
wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing,
inv_cov::Union{Matrix{Float64}, Nothing}=nothing)
if !(length(models) == length(ydata))
error("Dimension mismatch")
chisq(p,d) = get_chi(model,xdata,p,d,W)
chi2 = sum(fit.resid.^2)
par = fit_error(chisq,LsqFit.coef(fit),ydata,wpm,W,false)
# if the fit is correlated we have already C, no need to recompute it.
chiexp,pval = if length(C) !=0
GammaMethod.chiexp(chisq,LsqFit.coef(fit),value.(ydata),C,W),juobs.pvalue(chisq,chi2,LsqFit.coef(fit),ydata,W,C=C)
else
ADerrors.chiexp(chisq,LsqFit.coef(fit),ydata,wpm,W=W),juobs.pvalue(chisq,chi2,LsqFit.coef(fit),ydata,W)
end
N_sets = length(models) # number of datasets
x_flat = xdata[1] # unrolling xdata
y_flat = ydata[1] # unrolling ydata
chunk_idx = Vector{Vector{Int64}}(undef, N_sets)
j = 1
for i = 1:N_sets
isnothing(wpm) ? uwerr.(ydata[i]) : [uwerr(yaux, wpm) for yaux in ydata[i]]
if i > 1
y_flat = vcat(y_flat, ydata[i])
x_flat =vcat(x_flat, xdata[i])
end
stp = j + length(ydata[i]) - 1
chunk_idx[i] = collect(j:stp)
j = stp + 1
end
function flat_model(x_flat::Vector{Float64}, param::Vector{Float64})
fx = similar(x_flat)
for ii in 1:N_sets
mod = models[ii]
fx[chunk_idx[ii]] = mod(x_flat[chunk_idx[ii]], param)
if !isnothing(logfile)
uwerr.(par);
println(logfile, "juobs.fit_routine output:")
println(logfile, "\t\tFit routine in [$(xdata[1]),...,$(xdata[end])]")
println(logfile, "\t\tparameters :")
for p in par
println(logfile,"\t\t\t",p)
end
return fx
end
function flat_chisq(param, data)
if isnothing(inv_cov)
chi = 0.0
for ii in 1:N_sets
mod = models[ii]
chi += sum((data[chunk_idx[ii]] .- mod(x_flat[chunk_idx[ii]], param)).^2 ./ err.(y_flat[chunk_idx[ii]]).^2 )
end
return chi
else
# take pieces of covariance and build correlated chi2 piece by piec
# chi =0.0
# for ii in 1:N_sets
# mod = models[ii]
# cov_inv = juovs.inv_covar(ydata[ii])
# res = ydata[ii] .- mod(xdata[ii], param)
# chi += (reshape(res, 1,:) * cov_inv * res)[1]
# end
# return chi
# take whole covariance and multiply for whole vector of dat-mod(x,param)
res = Vector{uwreal}(undef, 0)
for ii in 1:N_sets
mod = models[ii]
res =vcat(res, data[chunk_idx[ii]] .- mod(x_flat[chunk_idx[ii]], param))
end
chi = reshape(res, 1,:) * inv_cov * res
return chi[1]
end
end
if isnothing(inv_cov)
@info("Uncorrelated fit")
fit = curve_fit(flat_model, x_flat, value.(y_flat), 1.0 ./ err.(y_flat).^2, guess_param )
(upar, chi2_exp) = isnothing(wpm) ? fit_error(flat_chisq, coef(fit) , y_flat) : fit_error(flat_chisq, coef(fit) , y_flat, wpm)
else
@info("Correlated fit")
fit = curve_fit(flat_model, x_flat, value.(y_flat), inv_cov, guess_param )
(upar, chi2_exp) = isnothing(wpm) ? fit_error(flat_chisq, coef(fit) , y_flat, W=inv_cov) : fit_error(flat_chisq, coef(fit) , y_flat, wpm, W=inv_cov)
end
println("chi2 from fit residue: ", sum(fit.resid.^2))
chisq = flat_chisq(coef(fit), y_flat)
if log
println("Converged param: ", fit.converged)
println("Chisq / chiexp: ", chisq, " / ", chi2_exp, " (dof: ", dof(fit),")")
chis2_corrected = (dof(fit)) * chisq / chi2_exp
println("Chisq corrected: ", chis2_corrected)
end
return upar, value(chisq), chi2_exp, dof(fit)
end
function fit_alg(f::Function, x::Union{Vector{Int64}, Vector{Float64}, Matrix{Float64}}, y::Vector{uwreal},
n::Int64, guess::Union{Float64, Vector{Float64}, Nothing}=nothing;
wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing)
isnothing(wpm) ? uwerr.(y) : [uwerr(y[i], wpm) for i in 1:length(y)]
W = 1 ./ err.(y) .^ 2
chisq = fit_defs(f,x,W)
#lb = [-Inf for i in 1:n]
#ub = [+Inf for i in 1:n]
if guess == nothing
p0 = [.5 for i in 1:n]
else
p0 = [guess; [1. for i in 1:n-length(guess)]]
#lb[1] = .9 * guess
#ub[1] = 1.1 * guess
end
fit = curve_fit(f,x,value.(y),W,p0)#,lower=lb,upper=ub)
chi2 = sum(fit.resid .^ 2)
isnothing(wpm) ? (up,chi_exp) = fit_error(chisq,coef(fit),y) : (up,chi_exp) = fit_error(chisq,coef(fit),y,wpm)
isnothing(wpm) ? pval = pvalue(chisq,chi2,value.(up),y) : pval = pvalue(chisq,chi2,value.(up),y,wpm=wpm)
return up, chi2, chi_exp, pval
end
function fit_alg(model::Function,x::Array{Float64},y::Array{uwreal},param::Int64,W::Matrix{Float64};
guess::Union{Float64, Vector{Float64}, Nothing}=nothing,
wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing)
if guess == nothing
p00 = [.5 for i in 1:param]
else
p00 = [guess; [1. for i in 1:param-length(guess)]]
#lb[1] = .9 * guess
#ub[1] = 1.1 * guess
println(logfile, "\t\tchi2: ", chi2)
println(logfile, "\t\tchiexp: ", chiexp)
println(logfile, "\t\tpvalue: ", pval)
end
chisq_corr(par,dat) = juobs.gen_chisq_correlated(model, x, W, par, dat)
fit = curve_fit(model,x,value.(y),W,p00)
up, chi_exp = fit_error(chisq_corr, coef(fit), y, wpm, W=W)
uwerr.(up)
chi2 = sum(fit.resid .^ 2)
pval = pvalue(chisq_corr, sum(fit.resid .^ 2), value.(up), y, W)
doff = dof(fit)
return up, chi2, chi_exp, pval, doff
return (par=par,chi2=chi2,chiexp=chiexp,pval=pval)
end
"""
check_size(M,n)
function fit_alg(f::Vector{Function}, x::Vector{Matrix{Float64}}, y::Vector{Vector{uwreal}},
n::Int64, guess::Union{Float64, Vector{Float64}, Nothing}=nothing;
wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing)
y_n = y[1]
x_n = x[1]
for i in 2:length(x)
y_n = vcat(y_n, y[i])
x_n = vcat(x_n, x[i])
end
idx = [collect(1:size(x[i],1)) for i in 1:length(x)]
[idx[i] = idx[i] .+ idx[i-1][end] for i in 2:length(idx)]
isnothing(wpm) ? [uwerr.(y[i]) for i in 1:length(y)] : [[uwerr(y[i][j], wpm) for j in 1:length(y[i])] for i in 1:length(y)]
W = [1 ./ err.(y[i]) .^ 2 for i in 1:length(y)]
chisq = (par, y_n) -> sum([sum((y_n[idx[i]] .- f[i](x[i], par)) .^ 2 .* W[i]) for i in 1:length(x)])
min_fun(t) = chisq(t, value.(y_n))
if guess == nothing
p0 = [.5 for i in 1:n]
else
p0 = [guess; [1. for i in 1:n-length(guess)]]
Check if `M` has sizes all equal to n or, if is empty
"""
function check_sizes(M,n)
if length(M) ==0 || all(size(M) .== n)
return true
end
sol = optimize(min_fun, p0, Optim.Options(g_tol=1e-8, iterations=1000000))
chi2 = min_fun(sol.minimizer)
isnothing(wpm) ? (up,chi_exp) = fit_error(chisq,sol.minimizer,y_n) : (up,chi_exp) = fit_error(chisq,sol.minimizer,y_n,wpm)
isnothing(wpm) ? pval = pvalue(chisq,chi2,value.(up),y_n) : pval = pvalue(chisq,chi2,value.(up),y_n,wpm=wpm)
return up, chi2, chi_exp, pval
return false
end
##CHECK THIS ONE, NOT WORKING GOOD
function fit_alg(f::Vector{Function}, x::Vector{Matrix{Float64}}, y::Vector{Vector{uwreal}}, W::Vector{Matrix{Float64}},
n::Int64, guess::Union{Float64, Vector{Float64}, Nothing}=nothing;
wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing)
y_n = y[1]
x_n = x[1]
for i in 2:length(x)
y_n = vcat(y_n, y[i])
x_n = vcat(x_n, x[i])
end
idx = [collect(1:size(x[i],1)) for i in 1:length(x)]
[idx[i] = idx[i] .+ idx[i-1][end] for i in 2:length(idx)]
isnothing(wpm) ? [uwerr.(y[i]) for i in 1:length(y)] : [[uwerr(y[i][j], wpm) for j in 1:length(y[i])] for i in 1:length(y)]
chisq = (par, y_n) -> sum([juobs.gen_chisq_correlated(f[i], x[i], W[i], par, y_n[i]) for i in 1:length(x)])
min_fun(t) = chisq(t, value.(y_n))
if guess == nothing
p0 = [.5 for i in 1:n]
function fit_routine(model::Function,
xdata::AbstractArray{ADerrors.uwreal},
ydata::AbstractArray{ADerrors.uwreal},
npar::Int64;
W::AbstractArray = Float64[],
wpm::AbstractDict = Dict{Int64,Vector{Float64}}(),
corr::Bool = true,
guess::AbstractArray = fill(0.5,npar),
logfile = nothing,
C::AbstractArray = Float64[])
Nx = length(xdata)
if (Nx != length(ydata))
error("[juobs] Error: xdata and ydata must have the same dimension")
end
Ndata = 2Nx; ## in total we have 2*lengt(xdata) data points
data = vcat(ydata,xdata)
if !check_sizes(W,Ndata)
error("[juobs] Error: W does not have the correct sizes")
end
if !check_sizes(C,Ndata)
error("[juobs] Error: C does not have the correct sizes")
end
if corr
C = length(C) ==0 ? GammaMethod.cov_AD(data) : C
W = length(W) ==0 ? LinearAlgebra.pinv(W) : W
else
p0 = [guess; [1. for i in 1:n-length(guess)]]
end
sol = optimize(min_fun, p0, Optim.Options(g_tol=1e-8, iterations=1000000))
chi2 = min_fun(sol.minimizer)
W_aux = Matrix{Float64}(undef, length(y_n), length(y_n))
for i in 1:length(y_n)
for j in 1:length(y_n)
if i <= length(y[1])
if j <= length(y[1])
W_aux[i,j] = W[1][i,j]
else
W_aux[i,j] = .0
end
else
if j <= length(y[1])
W_aux[i,j] = .0
else
W_aux[i,j] = W[2][i-length(y[1]),j-length(y[1])]
end
end
W = length(W) ==0 ? [1/yy.err^2 for yy in data] : W
end
f(q,p) = [i<=Nx ? model(q[i],p) : q[i-Nx] for i in 1:Ndata]
fit_func(x,p) = f(view(p,npar+1:npar+Nx),view(p,1:npar))
new_guess = [guess; value.(xdata)]
fit = fit_routine(fit_func,zeros(Ndata),data,npar+Nx,
guess=new_guess, wpm=wpm,corr=corr,
W=W,C=C,logfile=nothing)
if !isnothing(logfile)
uwerr.(fit.par)
println(logfile, "juobs.fit_routine output:")
println(logfile, "\t\txdata carries uncertainty")
println(logfile, "\t\tFit routine in [$(xdata[1].mean),...,$(xdata[end].mean)]")
println(logfile, "\t\tFit parameters:")
for i in 1:npar
println(logfile,"\t\t\t",fit.par[i])
end
println(logfile, "\t\tauxiliary fit parameter:")
println(logfile, "\t\t\txdata[i] \t q[i]")
for i in 1:Nx
println(logfile, "\t\t\t",xdata[i]," \t ", fit.par[npar+i])
end
println(logfile, "\t\tchi2: ", fit.chi2)
println(logfile, "\t\tchiexp: ", fit.chiexp)
println(logfile, "\t\tpvalue: ", fit.pval)
end
isnothing(wpm) ? (up,chi_exp) = fit_error(chisq,sol.minimizer,y_n,W=W_aux) : (up,chi_exp) = fit_error(chisq,sol.minimizer,y_n,wpm,W=W_aux)
isnothing(wpm) ? pval = pvalue(chisq,chi2,value.(up),y_n,W_aux) : pval = pvalue(chisq,chi2,value.(up),y_n,wpm=wpm,W_aux)
return up, chi2, chi_exp, pval
end
function fit_alg(f::Vector{Function}, x::Vector{Matrix{Float64}}, y::Vector{Vector{uwreal}},
n::Int64, lb::Vector{Float64}, ub::Vector{Float64}, guess::Union{Float64, Vector{Float64}, Nothing}=nothing;
wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing)
y_n = y[1]
x_n = x[1]
for i in 2:length(x)
y_n = vcat(y_n, y[i])
x_n = vcat(x_n, x[i])
end
idx = [collect(1:size(x[i],1)) for i in 1:length(x)]
[idx[i] = idx[i] .+ idx[i-1][end] for i in 2:length(idx)]
isnothing(wpm) ? [uwerr.(y[i]) for i in 1:length(y)] : [[uwerr(y[i][j], wpm) for j in 1:length(y[i])] for i in 1:length(y)]
W = [1 ./ err.(y[i]) .^ 2 for i in 1:length(y)]
chisq = (par, y_n) -> sum([sum((y_n[idx[i]] .- f[i](x[i], par)) .^ 2 .* W[i]) for i in 1:length(x)])
min_fun(t) = chisq(t, value.(y_n))
if guess == nothing
p0 = [.5 for i in 1:n]
else
p0 = [guess; [1. for i in 1:n-length(guess)]]
end
sol = optimize(min_fun, lb, ub, p0, Fminbox(NelderMead()), Optim.Options(g_tol=1e-8, iterations=1000000))
chi2 = min_fun(sol.minimizer)
isnothing(wpm) ? (up,chi_exp) = fit_error(chisq,sol.minimizer,y_n) : (up,chi_exp) = fit_error(chisq,sol.minimizer,y_n,wpm)
isnothing(wpm) ? pval = pvalue(chisq,chi2,value.(up),y_n) : pval = pvalue(chisq,chi2,value.(up),y_n,wpm=wpm)
return up, chi2, chi_exp, pval
end
function fit_defs(f::Function,x,W) ## uncorrelated fit
chisq(p,d) = sum((d .- f(x,p)).^2 .* W)
return chisq
return fit
end
function gen_chisq(f::Function, x::Array{<:Real}, err::Vector{Float64}) #constrained
chisq(par, dat) = sum((dat .- f(x,par)).^2 ./err.^2)
return chisq
end
function gen_chisq_correlated(f::Function, x::Array{<:Real}, covar_inv::Matrix, par, dat )
chi2 = 0.0
#delta = dat - ffit_routine(x, par)
#chi2 = (reshape(delta, 1,:) * covar_inv * delta )[1]
for i in eachindex(dat)
for j in eachindex(dat)
#println(f(x,par)[i])
#println(f(x,par)[j])
#println(f(x[j,:],par))
#println(f(x[i,:],par))
chi2 = chi2 + (dat[i] - f(x,par)[i]) * covar_inv[i,j] * (dat[j] - f(x,par)[j])
function fit_routine(models::AbstractArray{Function},
xdata::AbstractArray{<:AbstractArray},
ydata::AbstractArray{<:AbstractArray{ADerrors.uwreal}},
npar::AbstractArray{Int64};
W::AbstractArray = Float64[],
C::AbstractArray = Float64[],
wpm::AbstractDict = Dict{Int64,Vector{Float64}}(),
corr::Bool = true,
guess::AbstractArray{<:AbstractArray} = [fill(0.5,npar[i]) for i in eachindex(models)],
logfile = nothing,
)
Nmodel = length(models)
if Nmodel != length(xdata) != length(ydata)
error("[juobs] Error: you need the same number of model, xdata and ydata")
end
if !(length(W) == 0 || length(W)==Nmodel)
error("[juobs] Error: You need to pass a W matrix for model or none")
end
if !(length(C) == 0 || length(C)==Nmodel)
error("[juobs] Error: You need to pass a C matrix for model or none")
end
if length(npar) != Nmodel
error("[juobs] Error: You need to specify the number of parameter for every model")
end
if length(guess)!=Nmodel
error("[juobs] Error: You need to specify the inital guess for all model or for none")
end
ndata = length.(xdata)
if !all(length.(ydata).== ndata)
error("[juobs] Error: Mismatch in xdata and ydata. Make sure that the data corresponds")
end
Ntotal = sum(ndata);
X = vcat(xdata...)
Y = vcat(ydata...)
function make_matrix(M)
aux = zeros(Ntotal,Ntotal)
ie = 0
for m in 1:Nmodel
is = ie+1
ie += ndata[m];
aux[is:ie,is:ie] .= M[m]
end
return aux
end
#chisq(par, dat) = sum(
# transpose((dat .- f(x,par))) * covar_inv * (dat .- f(x,par))
#)
return chi2
end
function get_chi2(f::Function, data, ddata, par, Nalpha) #full
chi2 = 0.0
WW = length(W)==0 ? W : make_matrix(W);
CC = length(C)==0 ? C : make_matrix(C);
Ndata = div(length(data), Nalpha+1)
Npar = length(par) - Ndata * Nalpha
p = par[1:Npar]
for k = 1:Ndata
xx = [par[Npar + k + (i-1)*Ndata] for i = 1:Nalpha]
Cinv = zeros(Nalpha+1, Nalpha+1)
[Cinv[i, i] = 1 / ddata[k + (i-1)*Ndata]^2 for i = 1:Nalpha+1]
xx = [par[Npar + k + (i-1)*Ndata] for i = 1:Nalpha]
delta = [data[k + (i-1)*Ndata] - xx[i] for i = 1:Nalpha]
yy = f(xx', p)
push!(delta, data[k + Nalpha*Ndata] - yy[1])
chi2 += delta' * Cinv * delta
function Model(x,p)
res = zeros(Nmodel)
res[1] = models[1](x[1:ndata[1]],p[1:npar[1]])
for i in 2:Nmodel
rd = (ndata[i-1]+1):ndata[i]
rp = (npar[i-1]+1):npar[i]
res[i] = models[i](x[rd],p[rp])
end
return res
end
return chi2
end
function get_chi2_cov(f::Function, data, C, par, Nalpha) # full + cov
chi2 = 0.0
Ndata = div(length(data), Nalpha+1)
Npar = length(par) - Ndata * Nalpha
p = par[1:Npar]
Guess = vcat(guess...)
fit = fit_routine(Model,X,Y,sum(npar),guess = Guess,W=WW,C=CC,corr=corr,logfile=nothing,wpm=wpm)
for k = 1:Ndata
if det(C[k]) / prod(diag(C[k])) > 1e-6
Cinv = inv(C[k])
else
Cinv = zeros(Nalpha+1, Nalpha+1)
[Cinv[i, i] = 1 / C[k][i, i] for i = 1:Nalpha+1]
if !isnothing(logfile)
uwerr.(fit.par)
flag = typeof(xdata) <: AbstractArray{<:AbstractArray{ADerrors.uwreal}}
println(logfile, "juobs.fit_routine output:")
if flag
println(logfile, "\t\txdata carries uncertainty")
end
xx = [par[Npar + k + (i-1)*Ndata] for i = 1:Nalpha]
delta = [data[k + (i-1)*Ndata] - xx[i] for i = 1:Nalpha]
yy = f(xx', p)
push!(delta, data[k + Nalpha*Ndata] - yy[1])
chi2 += delta' * Cinv * delta
println(logfile, "\t\tGlobal fit with $Nmodel functions")
for nm in 1:Nmodel
println(logfile,"---Model $(nm)---")
println(logfile, "\t\tFit routine in [$(xdata[nm][1].mean),...,$(xdata[nm][end].mean)]")
println(logfile, "\t\tFit parameters:")
for i in 1:npar[nm]
off = nm ==1 ? 0 : npar[nm-1]
println(logfile,"\t\t\t",fit.par[off+i])
end
end
if flag
pritnln(logfile, "\t\tauxiliary fit parameter:")
println(logfile, "\t\t\txdata[m][i] \t q[i]")
for nm in 1:Nmodel, i in 1:npar[nm]
off = nm==1 ? 0 : npar[end]+npar[i-1]
println(logfile, "\t\t\t",xdata[nm][i]," \t ", fit.par[off+i])
end
end
println(logfile, "\t\tchi2: ", chi2)
println(logfile, "\t\tchiexp: ", chiexp)
println(logfile, "\t\tpvalue: ", pval)
end
return chi2
end
function get_covar(obs::Vector{uwreal}; wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing)
isnothing(wpm) ? uwerr.(obs) : [uwerr(yaux, wpm) for yaux in obs]
ens = vcat(ensembles.(obs)...)
tmp_fluc = mchist.(obs, ens)
fluc = mapreduce(permutedims, vcat, tmp_fluc)'
covar = fluc' * fluc ./ size(fluc)[1]^2
return covar
end
function get_covar_multi_id(obs::Vector{uwreal}; wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing)
isnothing(wpm) ? uwerr.(obs) : [uwerr(yaux, wpm) for yaux in obs]
id_set = getindex.(ensembles.(obs), 2) # hardcoded assuming that ens label is always the second entry in each ensembles(obs[i])
return fit
end
cov_tot = zeros(length(id_set), length(id_set))
count = 1
for (k,id) in enumerate(unique(id_set))
idx = findall(x-> x==id, id_set )
fluc_aux = mchist.(obs[idx], fill(id, length(idx)))
fluc = mapreduce(permutedims, vcat, fluc_aux)'
covar_set = fluc' * fluc ./ size(fluc)[1]^2
dim_tmp = size(covar_set,1)
cov_tot[count:dim_tmp+count-1, count:dim_tmp+count-1] = covar_set
count +=dim_tmp
end
return cov_tot
end
function inv_covar(obs::Vector{uwreal}; wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing)
cov = get_covar(obs, wpm=wpm)
cov_inv = inv(cov)
return (cov_inv' + cov_inv) / 2.
end
function inv_covar_multi_id(obs::Vector{uwreal}; wpm::Union{Dict{Int64,Vector{Float64}},Dict{String,Vector{Float64}}, Nothing}=nothing)
cov = get_covar_multi_id(obs, wpm=wpm)
cov_inv = inv(cov)
return (cov_inv' + cov_inv) / 2.
end =#
@doc raw"""
pvalue(chisq::Function,
......
src/juobs_types.jl
View file @ 8aa2abdb
##############UTILITY NAMES##############
##############UTILITY NAMES##############
#= STARTING AT 0
noise_name=['Z2', 'GAUSS', 'U1', 'Z4']
gamma_name = ['G0', 'G1', 'G2', 'G3',
......@@ -295,4 +296,4 @@ function Base.show(io::IO, a::Corr)
print(io, "$k = $f\t")
end
end
end
\ No newline at end of file
end
test/test_fit_routines.jl 0 → 100644
View file @ 8aa2abdb
using Revise, juobs, ADerrors
true_model(x) = 0.5 + 0.02*x +0.001*x^2
xdata = collect(0.0:0.1:1)
ydata = let
yaux = true_model.(xdata)
[uwreal(randn(1000)*0.0003.+y,"fit_test") for y in yaux]
end
uwerr.(ydata)
@.model(x,p) = p[1] + p[2]*x + p[3]*x^2
println("true model: 0.5 + 0.02x + 0.0001x^2")
println("fit model: p[1]+ p[2]x + p[3]x^2 ")
fit = fit_routine(model,xdata,ydata,3,logfile=stdout,corr=false);
xdata_s = [uwreal(randn(1000)*0.0003.+x,"xdata") for x in xdata]
fit2 = fit_routine(model,xdata_s,ydata,3,logfile=stdout,corr=false)
@.model2(x,p) = p[1] + (p[2] + p[3]*x)^2
Cy = [y.err^2 for y in ydata]
Cy = [i==j ? Cy[i] : 0 for i in eachindex(ydata), j in eachindex(ydata)]
W = [C==0 ? 0 : 1/C for C in Cy ]
fit3 = fit_routine([model,model2],[xdata,xdata],[ydata,ydata],[3,3],corr=false,logfile=stdout, W=[W,W],C=[Cy,Cy])
true
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment