% Import and merge GPS captain files into a clean single file minute binned


%% bin length
bin_size_days = 1 / 60 / 24;
parallel = Inf;
prctile_average = [2.5, 97.5];
prctile_detection = [2.5, 97.5];
bin_method = 'SB_ALL';

%%
pathfolder = '/Volumes/Data2/TaraEuropa/raw/GPS_captain';

list_csv = {dir(fullfile(pathfolder, 'GPScaptain_*')).name}';
GPScaptain_raw = table();
for i = progress(1:size(list_csv))
  data = readtable(fullfile(pathfolder, list_csv{i}));
  % lat
  foo = cellfun(@(x) strsplit(x, '-'), data.Latitude, 'un', 0);
  deg = str2double(cellfun(@(x) x{1}, foo, 'un', 0));
  dec_deg = str2double(cellfun(@(x) x{2}, foo, 'un', 0))/100/0.6;
  lat = deg + dec_deg;
  lat(strcmp(cellfun(@(x) x{3}, foo, 'un', 0), 'S')) = lat(strcmp(cellfun(@(x) x{3}, foo, 'un', 0), 'S')) * -1;
  % lon   
  foo = cellfun(@(x) strsplit(x, '-'), data.Longitude, 'un', 0);
  deg = str2double(cellfun(@(x) x{1}, foo, 'un', 0));
  dec_deg = str2double(cellfun(@(x) x{2}, foo, 'un', 0))/100/0.6;
  lon = deg + dec_deg;
  lon(strcmp(cellfun(@(x) x{3}, foo, 'un', 0), 'W')) = lon(strcmp(cellfun(@(x) x{3}, foo, 'un', 0), 'W')) * -1;
  data = addvars(data, datetime(data.Year,data.Month,data.Day,data.Hour,data.Minute,data.Second), lat, lon, ...
    'NewVariableNames', {'dt','lat','lon'}, 'Before', 'Latitude');
  data = removevars(data, {'Latitude','Longitude','Year','Month','Day','Hour','Minute','Second'});
  GPScaptain_raw = [GPScaptain_raw; data];
end

figure; scatter(GPScaptain_raw.dt, GPScaptain_raw.lat)
save(fullfile(pathfolder, 'GPScaptain_raw.mat'), 'GPScaptain_raw')

GPScaptain_raw.dt = datenum(GPScaptain_raw.dt);
GPScaptain_raw_binned = [];
for d = floor(min(GPScaptain_raw.dt)):floor(max(GPScaptain_raw.dt))
  fprintf('\t\t%s', datestr(d)); tic;
  sel = d <= GPScaptain_raw.dt & GPScaptain_raw.dt < d + 1;
  if any(sel)
    GPScaptain_raw_binned = [GPScaptain_raw_binned; binTable(GPScaptain_raw(sel,:), bin_size_days, ...
      bin_method, prctile_detection, prctile_average, false, parallel, false)];
  end
  t = toc; fprintf('  %1.3f s\n', t);
end
GPScaptain_raw_binned.dt = datetime(GPScaptain_raw_binned.dt, 'ConvertFrom', 'datenum');
GPScaptain_raw_binned = round_timestamp(GPScaptain_raw_binned);

figure; scatter(GPScaptain_raw_binned.dt, GPScaptain_raw_binned.lat)
save(fullfile(pathfolder, 'GPScaptain_raw_binned.mat'), 'GPScaptain_raw_binned')

%% reformat GPS_captain2
load(fullfile(pathfolder, 'GPS_captain2.mat'))
GPS_captain2 = table();
GPS_captain2.dt = datetime(numdate, 'ConvertFrom', 'datenum');
GPS_captain2.lat = lat;
GPS_captain2.lon = lon;
GPS_captain2 = sortrows(GPS_captain2, 'dt');

figure; scatter(GPS_captain2.dt, GPS_captain2.lat)
save(fullfile(pathfolder, 'GPS_captain2.mat'), 'GPS_captain2')

GPS_captain2.dt = datenum(GPS_captain2.dt);
GPS_captain2_binned = [];
for d = floor(min(GPS_captain2.dt)):floor(max(GPS_captain2.dt))
  fprintf('\t\t%s', datestr(d)); tic;
  sel = d <= GPS_captain2.dt & GPS_captain2.dt < d + 1;
  if any(sel)
    GPS_captain2_binned = [GPS_captain2_binned; binTable(GPS_captain2(sel,:), bin_size_days, ...
      bin_method, prctile_detection, prctile_average, false, parallel, false)];
  end
  t = toc; fprintf('  %1.3f s\n', t);
end
GPS_captain2_binned.dt = datetime(GPS_captain2_binned.dt, 'ConvertFrom', 'datenum');
GPS_captain2_binned = round_timestamp(GPS_captain2_binned);

figure; scatter(GPS_captain2_binned.dt, GPS_captain2_binned.lat)
save(fullfile(pathfolder, 'GPS_captain2_binned.mat'), 'GPS_captain2_binned')

%% Position_GPs_tara_Lorient_cadiz_workedup.mat
%%%%% CRAP %%%%
% load(fullfile(pathfolder, 'Position_GPs_tara_Lorient_cadiz_workedup.mat'))
% Position_GPs_tara_Lorient_cadiz_workedup = table();
% Position_GPs_tara_Lorient_cadiz_workedup.dt = datetime(datetime_gps_cadiz, 'ConvertFrom', 'datenum');
% Position_GPs_tara_Lorient_cadiz_workedup.lat = lat;
% Position_GPs_tara_Lorient_cadiz_workedup.lon = lon;
% Position_GPs_tara_Lorient_cadiz_workedup = sortrows(Position_GPs_tara_Lorient_cadiz_workedup, 'dt');
% save(fullfile(pathfolder, 'Position_GPs_tara_Lorient_cadiz_workedup.mat'), 'Position_GPs_tara_Lorient_cadiz_workedup')
% 
% figure; scatter(Position_GPs_tara_Lorient_cadiz_workedup.dt, Position_GPs_tara_Lorient_cadiz_workedup.lat)
% save(fullfile(pathfolder, 'Position_GPs_tara_Lorient_cadiz_workedup.mat'), 'Position_GPs_tara_Lorient_cadiz_workedup')
% 
% Position_GPs_tara_Lorient_cadiz_workedup.dt = datenum(Position_GPs_tara_Lorient_cadiz_workedup.dt);
% Position_GPs_tara_Lorient_cadiz_workedup_binned = [];
% for d = floor(min(Position_GPs_tara_Lorient_cadiz_workedup.dt)):floor(max(Position_GPs_tara_Lorient_cadiz_workedup.dt))
%   fprintf('\t\t%s', datestr(d)); tic;
%   sel = d <= Position_GPs_tara_Lorient_cadiz_workedup.dt & Position_GPs_tara_Lorient_cadiz_workedup.dt < d + 1;
%   if any(sel)
%     Position_GPs_tara_Lorient_cadiz_workedup_binned = [Position_GPs_tara_Lorient_cadiz_workedup_binned; binTable(Position_GPs_tara_Lorient_cadiz_workedup(sel,:), bin_size_days, ...
%       bin_method, prctile_detection, prctile_average, false, parallel, false)];
%   end
%   t = toc; fprintf('  %1.3f s\n', t);
% end
% Position_GPs_tara_Lorient_cadiz_workedup_binned.dt = datetime(Position_GPs_tara_Lorient_cadiz_workedup_binned.dt, 'ConvertFrom', 'datenum');
% Position_GPs_tara_Lorient_cadiz_workedup_binned = round_timestamp(Position_GPs_tara_Lorient_cadiz_workedup_binned);
% 
% figure; scatter(Position_GPs_tara_Lorient_cadiz_workedup_binned.dt, Position_GPs_tara_Lorient_cadiz_workedup_binned.lat)
% save(fullfile(pathfolder, 'Position_GPs_tara_Lorient_cadiz_workedup_binned.mat'), 'Position_GPs_tara_Lorient_cadiz_workedup_binned')

%% Position_GPS_TARA.txt
Position_GPS_TARA = readtable(fullfile(pathfolder, 'Position_GPS_TARA.txt'));
Position_GPS_TARA.Properties.VariableNames = {'number','lat_str','ns','sep','lon_str','we','date','time','heading','sst_batos', 'sst_unit'};
% datetime
dt = datetime(Position_GPS_TARA.date + Position_GPS_TARA.time, 'Format', 'dd-MMM-uuuu HH:mm:ss');

% lat
Position_GPS_TARA.lat_str = strrep(strrep(Position_GPS_TARA.lat_str, '"', '.'), "'", '.');
foo = cellfun(@(x) strsplit(x, '°'), Position_GPS_TARA.lat_str, 'un', 0);
deg = str2double(cellfun(@(s) s{1}, foo, 'un', 0));
dec_deg = str2double(cellfun(@(x) x{2}, foo, 'un', 0))/100/0.6;
lat = deg + dec_deg;
lat(strcmp(Position_GPS_TARA.ns, 'S')) = lat(strcmp(Position_GPS_TARA.ns, 'S')) * -1;
% lon
Position_GPS_TARA.lon_str = strrep(strrep(Position_GPS_TARA.lon_str, '"', '.'), "'", '.');
foo = cellfun(@(x) strsplit(x, '°'), Position_GPS_TARA.lon_str, 'un', 0);
deg = str2double(cellfun(@(s) s{1}, foo, 'un', 0));
dec_deg = str2double(cellfun(@(x) x{2}, foo, 'un', 0))/100/0.6;
lon = deg + dec_deg;
lon(strcmp(Position_GPS_TARA.we, 'W')) = lon(strcmp(Position_GPS_TARA.we, 'W')) * -1;
Position_GPS_TARA = addvars(Position_GPS_TARA, dt, lat, lon, ...
  'NewVariableNames', {'dt','lat','lon'}, 'Before', 'number');
Position_GPS_TARA = removevars(Position_GPS_TARA, {'number','lat_str','ns','sep','lon_str','we','date','time','sst_unit'});
Position_GPS_TARA.heading = str2double(strrep(Position_GPS_TARA.heading, '°', ''));

figure; scatter(Position_GPS_TARA.dt, Position_GPS_TARA.lat)
save(fullfile(pathfolder, 'Position_GPS_TARA.mat'), 'Position_GPS_TARA')

Position_GPS_TARA.dt = datenum(Position_GPS_TARA.dt);
Position_GPS_TARA_binned = [];
for d = floor(min(Position_GPS_TARA.dt)):floor(max(Position_GPS_TARA.dt))
  fprintf('\t\t%s', datestr(d)); tic;
  sel = d <= Position_GPS_TARA.dt & Position_GPS_TARA.dt < d + 1;
  if any(sel)
    Position_GPS_TARA_binned = [Position_GPS_TARA_binned; binTable(Position_GPS_TARA(sel,:), bin_size_days, ...
      bin_method, prctile_detection, prctile_average, false, parallel, false)];
  end
  t = toc; fprintf('  %1.3f s\n', t);
end
Position_GPS_TARA_binned.dt = datetime(Position_GPS_TARA_binned.dt, 'ConvertFrom', 'datenum');
Position_GPS_TARA_binned = round_timestamp(Position_GPS_TARA_binned);

figure; scatter(Position_GPS_TARA_binned.dt, Position_GPS_TARA_binned.lat)
Position_GPS_TARA_binned(Position_GPS_TARA_binned.dt == datetime(2023,5,15,7,7,0), :) = [];
figure; scatter(Position_GPS_TARA_binned.dt, Position_GPS_TARA_binned.lat)

save(fullfile(pathfolder, 'Position_GPS_TARA_binned.mat'), 'Position_GPS_TARA_binned')

%% TARA_track_Lorient_Cadiz.txt
TARA_track_Lorient_Cadiz = readtable(fullfile(pathfolder, 'TARA_track_Lorient_Cadiz.txt'));
% datetime
dt = datetime(TARA_track_Lorient_Cadiz.UTC, 'Format', 'dd-MMM-uuuu HH:mm:ss');

% GPS
TARA_track_Lorient_Cadiz.Position = strrep(strrep(TARA_track_Lorient_Cadiz.Position, '"', '.'), "'", '.');
foo_position = cellfun(@(x) strsplit(x, ' '), TARA_track_Lorient_Cadiz.Position, 'un', 0);

% lat
lat_str = cellfun(@(x) x{1}, foo_position, 'un', 0);
ns = cellfun(@(x) x{2}, foo_position, 'un', 0);
foo = cellfun(@(x) strsplit(x, '°'), lat_str, 'un', 0);
deg = str2double(cellfun(@(s) s{1}, foo, 'un', 0));
dec_deg = str2double(cellfun(@(x) x{2}, foo, 'un', 0))/100/0.6;
lat = deg + dec_deg;
lat(strcmp(ns, 'S')) = lat(strcmp(ns, 'S')) * -1;
% lon
lon_str = cellfun(@(x) x{4}, foo_position, 'un', 0);
we = cellfun(@(x) x{5}, foo_position, 'un', 0);
foo = cellfun(@(x) strsplit(x, '°'), lon_str, 'un', 0);
deg = str2double(cellfun(@(s) s{1}, foo, 'un', 0));
dec_deg = str2double(cellfun(@(x) x{2}, foo, 'un', 0))/100/0.6;
lon = deg + dec_deg;
lon(strcmp(we, 'W')) = lon(strcmp(we, 'W')) * -1;
TARA_track_Lorient_Cadiz = addvars(TARA_track_Lorient_Cadiz, dt, lat, lon, ...
  'NewVariableNames', {'dt','lat','lon'}, 'Before', 'n_');
TARA_track_Lorient_Cadiz = renamevars(TARA_track_Lorient_Cadiz, {'CapVrai','Temp_ratureDeL_eau'}, {'heading','sst_batos'});
TARA_track_Lorient_Cadiz.heading = str2double(strrep(TARA_track_Lorient_Cadiz.heading, '°', ''));
TARA_track_Lorient_Cadiz = removevars(TARA_track_Lorient_Cadiz, {'n_','Position','UTC'});

figure; scatter(TARA_track_Lorient_Cadiz.dt, TARA_track_Lorient_Cadiz.lat)
save(fullfile(pathfolder, 'TARA_track_Lorient_Cadiz.mat'), 'TARA_track_Lorient_Cadiz')

TARA_track_Lorient_Cadiz.dt = datenum(TARA_track_Lorient_Cadiz.dt);
TARA_track_Lorient_Cadiz_binned = [];
for d = floor(min(TARA_track_Lorient_Cadiz.dt)):floor(max(TARA_track_Lorient_Cadiz.dt))
  fprintf('\t\t%s', datestr(d)); tic;
  sel = d <= TARA_track_Lorient_Cadiz.dt & TARA_track_Lorient_Cadiz.dt < d + 1;
  if any(sel)
    TARA_track_Lorient_Cadiz_binned = [TARA_track_Lorient_Cadiz_binned; binTable(TARA_track_Lorient_Cadiz(sel,:), bin_size_days, ...
      bin_method, prctile_detection, prctile_average, false, parallel, false)];
  end
  t = toc; fprintf('  %1.3f s\n', t);
end
TARA_track_Lorient_Cadiz_binned.dt = datetime(TARA_track_Lorient_Cadiz_binned.dt, 'ConvertFrom', 'datenum');
TARA_track_Lorient_Cadiz_binned = round_timestamp(TARA_track_Lorient_Cadiz_binned);

figure; scatter(TARA_track_Lorient_Cadiz_binned.dt, TARA_track_Lorient_Cadiz_binned.lat)
save(fullfile(pathfolder, 'TARA_track_Lorient_Cadiz_binned.mat'), 'TARA_track_Lorient_Cadiz_binned')

%% merge GPS into single file minute binned

load(fullfile(pathfolder, 'GPScaptain_raw_binned.mat'), 'GPScaptain_raw_binned')
load(fullfile(pathfolder, 'GPS_captain2_binned.mat'), 'GPS_captain2_binned')
load(fullfile(pathfolder, 'Position_GPS_TARA_binned.mat'), 'Position_GPS_TARA_binned')
load(fullfile(pathfolder, 'TARA_track_Lorient_Cadiz_binned.mat'), 'TARA_track_Lorient_Cadiz_binned')

% create dt vector
merged_GPScaptain = table();
merged_GPScaptain.dt = (min([GPScaptain_raw_binned.dt; GPS_captain2_binned.dt; Position_GPS_TARA_binned.dt; TARA_track_Lorient_Cadiz_binned.dt]):minutes(1):...
  max([GPScaptain_raw_binned.dt; GPS_captain2_binned.dt; Position_GPS_TARA_binned.dt; TARA_track_Lorient_Cadiz_binned.dt]))';
% merge lat, lon, sst, sss
replace_consecutive_nan = 1*60; % 1h
merged_GPScaptain = merge_timeseries(merged_GPScaptain, GPScaptain_raw_binned, {'lat', 'lon'});
merged_GPScaptain = merge_timeseries(merged_GPScaptain, GPS_captain2_binned, {'lat', 'lon'});
merged_GPScaptain = merge_timeseries(merged_GPScaptain, Position_GPS_TARA_binned, {'lat', 'lon'});
merged_GPScaptain = merge_timeseries(merged_GPScaptain, TARA_track_Lorient_Cadiz_binned, {'lat', 'lon'}, '', replace_consecutive_nan);

figure; scatter(merged_GPScaptain.dt, merged_GPScaptain.lat)
merged_GPScaptain(merged_GPScaptain.dt == datetime(2023,4,5,5,45,0), :) = [];
figure; scatter(merged_GPScaptain.dt, merged_GPScaptain.lat)

filename = sprintf('TaraEuropa_merged_GPScaptain_v%s.mat', datetime('today', 'Format', 'yyyyMMdd'));
save(fullfile(pathfolder, filename), 'merged_GPScaptain')






