Predicting Mortality of ICU Patients: The PhysioNet/Computing in Cardiology Challenge 2012 1.0.0

File: <base>/sources/alistairewj_at_gmail.com/entry4/pnPreprocess.m (23,593 bytes)
function [ data ] = pnPreprocess(data)
%PNPREPROCESS	Output data after preprocessing
%   [ data ] = pnPreprocess(data)
% Primary variables:
%   data      - Cell array data, rows = # of patients
%   dataDesc  - Cell array of data fields/descriptions
%   dataFixed - Cell array of demographic data fields/descriptions
%   tmp       - Data for a single parameter, e.g. 'HR'
%   idxRem    - Index for data to be removed at end of the loop
%   idxManip  - Index for data to be manipulated some how in switch block
%   high/low  - Extracted data which is manipulated and re-imputed into tmp

%	Copyright 2012 Alistair Johnson

%	$LastChangedBy$
%	$LastChangedDate$
%	$Revision$
%	Originally written on PCWIN64 by Alistair Johnson, 25-Apr-2012 01:37:49
%	Contact: alistairewj@gmail.com

verboseFlag = false;

data_processed = data;
[dataDesc,dataFixed]  = pnDataDescriptions();

%=== Loop through the 3 fixed, demographic fields
for k=1:size(dataFixed,1)
    fn = dataFixed{k,1};
    if verboseFlag; fprintf('\n%%=== %s ===%%\n', fn); end;
    
    [tmp,idx] = pnExtractField(data_processed,fn);
    
    %=== Reset delete indices
    idxRem = [];
    switch fn
        case 'RecordID'
            continue;
        case 'Age'
            idxManip = cellfun(@(x) x>100, tmp(:,4),'UniformOutput',false);
            high = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            high = cellfun(@(x) x*0+105, high, 'UniformOutput',false);
            
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = high{m};
            end
            if verboseFlag; fprintf('Replaced %2.0f values of 200 with 105.\n',sum(cell2mat(idxManip))); end;
            %=== Analyze residuals of data, check for bias
            % idxManip = cellfun(@(x) numel(x), tmp(:,4),'UniformOutput',false);
            % tmpDataAnalyze = cellfun(@(x,y) x(1:y-1) - x(y), tmp(:,4), idxManip, 'UniformOutput',false);
            % tmpDataAnalyze = cell2mat(tmpDataAnalyze(cellfun(@(x) ~isempty(x), tmpDataAnalyze)));
            % hist(tmpDataAnalyze,-10:1:10); xlabel('Age (1:end-1) - Age(end)');
            
            idxManip = cellfun(@(x) [true(numel(x)-1,1);false], tmp(:,4),'UniformOutput',false);
            %=== Impute 0s for vector values
            tmp(:,4) = cellfun(@(x,y) x-x.*y, tmp(:,4), idxManip, 'UniformOutput',false);
            
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq'); % index ==0 for removal
            
            if verboseFlag; fprintf('Deleted %2.0f vector values, leaving only the end value.\n',N); end;
        case 'Gender'
            idxManip = cellfun(@(x) (x==-1), tmp(:,4),'UniformOutput',false);
            high = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            high = cellfun(@(x) NaN, high, 'UniformOutput',false);
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = high{m};
            end
            if verboseFlag; fprintf('Replaced %2.0f values which were -1 with NaN.\n',sum(cell2mat(idxManip))); end;
        case 'Height'
            % 1 centimetre = 0.393700787 inches
            % 1 foot = 30.48 centimetres
            % 1 inch = 2.54 centimetres
            idxManip = cellfun(@(x) x==-1, tmp(:,4),'UniformOutput',false);
            low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            low = cellfun(@(x) NaN, low, 'UniformOutput',false);
            
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = low{m};
            end
            if verboseFlag; fprintf('Replaced %2.0f values which were -1 with NaN.\n',sum(~cell2mat(idxManip))); end;
            
            
            idxManip = cellfun(@(x) x<10, tmp(:,4),'UniformOutput',false);
            low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            low = cellfun(@(x) x*100, low, 'UniformOutput',false);
            
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = low{m};
            end
            if verboseFlag; fprintf('Multipled %2.0f values of <10 by 100 (1.8->180).\n',sum(cell2mat(idxManip))); end;
            
            idxManip = cellfun(@(x) x<25, tmp(:,4),'UniformOutput',false);
            low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            low = cellfun(@(x) x*10, low, 'UniformOutput',false);
            
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = low{m};
            end
            if verboseFlag; fprintf('Multipled %2.0f values of <25 by 10 (18->180).\n',sum(cell2mat(idxManip))); end;
            
            idxManip = cellfun(@(x) x<100, tmp(:,4),'UniformOutput',false);
            low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            low = cellfun(@(x) x*2.54, low, 'UniformOutput',false);
            
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = low{m};
            end
            if verboseFlag; fprintf('Multipled %2.0f values of <100 by 2.2 (81.8->180).\n',sum(cell2mat(idxManip))); end;
            
            
            idxManip = cellfun(@(x) x>1000, tmp(:,4),'UniformOutput',false);
            high = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            high = cellfun(@(x) x*0.1, high, 'UniformOutput',false);
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = high{m};
            end
            if verboseFlag; fprintf('Multipled %2.0f values of >1000 by 0.1 (1800->180).\n',sum(cell2mat(idxManip))); end;
            
            idxManip = cellfun(@(x) x>250, tmp(:,4),'UniformOutput',false);
            high = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            high = cellfun(@(x) x/2.54, high, 'UniformOutput',false);
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = high{m};
            end
            if verboseFlag; fprintf('Multipled %2.0f values of >250 by 0.3937 (400->157).\n',sum(cell2mat(idxManip))); end;
            
    end
    %=== Replace data with tmp
    for m=2:size(data,2)
        data_processed(:,m) = cellfun(@pnPreprocessReplaceData, data_processed(:,m), tmp(:,m), idx, 'UniformOutput', false);
    end
    
    %=== Delete entries flagged for deletion
    if ~isempty(idxRem)
        for m=2:size(data,2)
            data_processed(:,m) = cellfun(@pnPreprocessDeleteData, data_processed(:,m), idxRem, 'UniformOutput', false);
        end
    end
end

%=== Loop through all the fields, and process each appropriately
for k=1:size(dataDesc,1)
    fn = dataDesc{k,1};
    if verboseFlag; fprintf('\n%%=== %s ===%%\n', fn);
    [tmp,idx] = pnExtractField(data_processed,fn);
    idxRem=[];
    switch fn
        case 'Albumin'
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'ALP'
            %=== Possible conversion errors in ALP, ALT, AST:
            % Listed as measured in IU
            % 1 Katal = 60,000,000 IU
            % 1 micro Katal = 60 IU
            % 1 nano Katal = 0.060 IU
            % *** micro kats are sometimes used.
            
            % Reference range: 30 to 120 IU/L
            if verboseFlag; fprintf('No preprocessing performed - Note: highly skewed. Dependent on age and gender.\n'); end;
        case 'ALT'
            % Reference range: 7 to 56 IU/L
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'AST'
            % Reference range: 5 to 40 IU/L
            % Very heavy tailed: Normal range 5 to 40 IU/L, 10%% data > 1000 IU/L.
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'Bilirubin'
            % Reference range: 0.2 to 1.2 mg/dL
            % 1 mg/dL	== 17.1	µmol/L
            % It is possible to have >36 mg/dL in cases of transplant, etc.
            % Can't unilaterally fix this.
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'BUN'
            % Reference range: 10-20 mg/dl (3.6-71 mmol/liter)
            % 1 mg/dL	== 0.357 mmol/L
            % Can't really convert anything here.
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq');
            
            if verboseFlag; fprintf('Deleted %2.0f values which were 0.\n',N); end;
            %             for m=2:size(tmp,2)
            %                 tmp(:,m) = cellfun(@(x,y) x(y), tmp(:,m), idxManip,'UniformOutput',false);
            %             end
        case 'Cholesterol'
            % Reference range:
            %   Desirable           <200 mg/dl      <5.17 mmol/L
            %   Borderline high      200-239 mg/dl	 5.17-6.18 mmol/L
            %   High                ≥240 mg/dl      ≥6.18 mmol/L
            % 1 mg/dL	== 0.0259	mmol/L
            % Nothing bad here.
            if verboseFlag; fprintf('No preprocessing needed.\n'); end;
        case 'Creatinine'
            % Reference range: 0.6-1.3  mg/dl
            %   <1.5 mg/dl (NEJM)
            %   1 mg/dL	== 88.4	µmol/L
            %===  ~6 is a reasonable maximum, 7.6 is very confident
            % Seems OK.
            if verboseFlag; fprintf('No preprocessing needed.\n'); end;
        case 'DiasABP'
            %=== First, delete '0's since we don't know if it was badly
            % converted or missing
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq');
            if verboseFlag; fprintf('Deleted %2.0f values which were 0.\n',N); end;
            
            [idxRem2,N] = pnPreprocessRemovalIndices(tmp, idx, -1, 'eq');
            if verboseFlag; fprintf('Deleted %2.0f values which were -1.\n',N); end;
            
            idxRem = cellfun(@(x,y) x | y, idxRem, idxRem2, 'UniformOutput',false);
            
            idxManip = cellfun(@(x) x>170, tmp(:,4),'UniformOutput',false);
            idxManip2 = cellfun(@(x) x<200, tmp(:,4),'UniformOutput',false);
            idxManip = cellfun(@(x) sum(x), idxManip);
            idxManip2 = cellfun(@(x) sum(~x), idxManip2);
            if verboseFlag; fprintf('There exist %2.0f values between 170-200, which were left as is.\n',sum(idxManip)-sum(idxManip2)); end;
            
            [idxRem2, N] = pnPreprocessRemovalIndices(tmp, idx, 200, 'gt');
            [idxRem] = cellfun(@(x,y) x | y, idxRem, idxRem2, 'UniformOutput',false);
            if verboseFlag; fprintf('Deleted %2.0f values above 200.\n',N); end;
        case 'FiO2'
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'GCS'
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'Glucose'
            % Reference ranges:
            % Fasting
            %   Normal               75-115 mg/dl	 4.2-6.4 mmol/L
            %   Diabetes mellitus	>125 mg/dl      >7.0 mmol/L
            %   2 Hr post-meal      <120 mg/dl      <6.7 mmol/L
            % 1 mg/dL	== 0.0555	mmol/L
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'HCO3'
            % Reference ranges: 21-30 mEq/L	21-28 mmol/L
            % 1 mEq/L == 1 mmol/L
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'HCT'
            % Reference range: 41.0-53.0%
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'HR'
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq');
            if verboseFlag; fprintf('Deleted %2.0f values which were 0.\n',N); end;
            
            [idxRem2,N] = pnPreprocessRemovalIndices(tmp, idx, 300, 'eq');
            idxRem = cellfun(@(x,y) x | y, idxRem, idxRem2, 'UniformOutput',false); % Combine removal indices
            if verboseFlag; fprintf('Deleted %2.0f values which were 300.\n',N); end;
        case 'K'
            % Reference ranges: 3.5-5.0 mEq/L	3.5-5.0 mmol/L
            % 1 mEq/L == 1 mmol/L
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'Lactate' % mmol/L
            % Reference ranges: 5-15 mg/dl	0.6-1.7 mmol/liter
            % 1 mg/dL	== 0.111	mmol/L
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'Mg' % mmol/L
            % Reference ranges: 1.8-3.0 mg/dl	0.8-1.2 mmol/L
            %1 mg/dL	== 0.411 mmol/L
            %1 mEq/L	== 0.50	 mmol/L
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'MAP' % mmHg
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
            if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
        case 'MechVent'
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'Na' % mEq/L
            % Reference ranges: 136-145 mEq/L	136-145 mmol/L
            % 1 mEq/L == 1 mmol/L
            %=== Interesting spike at 150, possible rounding bias?
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'NIDiasABP' % mmHg
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
            if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
        case 'NIMAP' % mmHg
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
            if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
            
        case 'NISysABP' % mmHg
            % Values below 1??
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'eq');
            if verboseFlag; fprintf('Deleted %2.0f values which less than 1.\n',N); end;
            
        case 'PaCO2' % mmHg
            % Reference range:
            % 4.7-6.0 kPa 
            % 35-45 mmHg
            % 1 kPa == 7.5006 mmHg
            
            % Values below 1 must be wrong...
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
            if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
%             idxManip = cellfun(@(x) x<1, tmp(:,4),'UniformOutput',false);
%             low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
%             low = cellfun(@(x) x*100, low, 'UniformOutput',false);
%             
%             for m=1:size(tmp,1)
%                 tmp{m,4}(idxManip{m}) = low{m};
%             end
        case 'PaO2' % mmHg
            % Reference range:
            % 11-13 kPa 
            % 75-100 mmHg
            % 1 kPa == 7.5006 mmHg
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq');
            if verboseFlag; fprintf('Deleted %2.0f values which were 0.\n',N); end;
            
            idxManip = cellfun(@(x) x<10, tmp(:,4),'UniformOutput',false);
            low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            low = cellfun(@(x) x*7.5006, low, 'UniformOutput',false);
            
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = low{m};
            end
            if verboseFlag; fprintf('Replaced %2.0f values which were kPa, not mmHg.\n',sum(cell2mat(idxManip))); end;
        case 'pH'
            % Reference range:
            %   7.34-7.45 units
            
            %=== This is a mess
            % There are 5 values between 94-100, not sure why. Probably put
            % in the wrong field? Doesn't match with an [H+] conversion
            idxManip = cellfun(@(x) (x>0.65 & x<0.8), tmp(:,4),'UniformOutput',false);
            low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            low = cellfun(@(x) x*7.5006, low, 'UniformOutput',false);
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = low{m};
            end
            if verboseFlag; fprintf('Multiplied %2.0f values by 10 (0.7->7).\n',sum(cell2mat(idxManip))); end;
            
            idxManip = cellfun(@(x) (x>65 & x<80), tmp(:,4),'UniformOutput',false);
            low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            low = cellfun(@(x) x*0.1, low, 'UniformOutput',false);
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = low{m};
            end
            if verboseFlag; fprintf('Multiplied %2.0f values by 0.1 (70->7).\n',sum(cell2mat(idxManip))); end;
            
            idxManip = cellfun(@(x) (x>650 & x<800), tmp(:,4),'UniformOutput',false);
            low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            low = cellfun(@(x) x*0.01, low, 'UniformOutput',false);
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = low{m};
            end
            if verboseFlag; fprintf('Multiplied %2.0f values by 0.01 (700->7).\n',sum(cell2mat(idxManip))); end;
            
            
            %=== Values <6.5 and >0.8 are deleted
            % Note: (there are no values around 0.6-0.75)
            [idxRem] = pnPreprocessRemovalIndices(tmp, idx, 0.8, 'gt');
            [idxRem2] = pnPreprocessRemovalIndices(tmp, idx, 6.5, 'lt');
            idxRem = cellfun(@(x,y) x & y, idxRem, idxRem2, 'UniformOutput',false);
            N = sum(cell2mat(cellfun(@(x) sum(x), idxRem, 'UniformOutput', false)));
            if verboseFlag; fprintf('Deleted %2.0f values which were between [0.8,6.5].\n',N); end;
            
            %=== Values >80 & <650 are deleted
            % Note: (there are no values around 0.6-0.75)
            [idxRem3] = pnPreprocessRemovalIndices(tmp, idx, 80, 'gt');
            [idxRem4] = pnPreprocessRemovalIndices(tmp, idx, 650, 'lt');
            idxRem = cellfun(@(x,y,z) (x & y) | z, idxRem3, idxRem4, idxRem, 'UniformOutput',false);
            N = sum(cell2mat(cellfun(@(x,y) sum(x&y), idxRem3, idxRem4, 'UniformOutput', false)));
            if verboseFlag; fprintf('Deleted %2.0f values which were between [80,650].\n',N); end;
        case 'Platelets' % cells/nL
            % Reference range: 150-350 (10e3)/mm^3
            % 1 (10e3)/µL == 1 (10e3)/mm^3 == 1/nL
            % Errors are probably in orders of 1000
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'RespRate' % bpm
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq');
            if verboseFlag; fprintf('Deleted %2.0f values which were 0.\n',N); end;
        case 'SaO2' % %
            % Reference range: 94-100
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'SysABP' % mmHg
            % Reference range:
            % 10-14     kPa
            % 75-105	mmHg
            % 1 kPa == 7.5006 mmHg
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
            %             for m=2:size(tmp,2)
            %                 tmp(:,m) = cellfun(@(x,y) x(y), tmp(:,m), idxManip,'UniformOutput',false);
            %             end
            if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
        case 'Temp'
            %=== Check for Farenheit measurements and incorrect conversions
            %=== There are 129 values which are too low...
            %=== First, there are 39 '0's, which should be set to deleted,
            %since we don't know if it was badly converted or missing
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'le');
            if verboseFlag; fprintf('Deleted %2.0f values which were 0.\n',N); end;
            
            [idxRem2,N] = pnPreprocessRemovalIndices(tmp, idx, -17, 'lt');
            if verboseFlag; fprintf('Deleted %2.0f values which were -17.8 (i.e. incorrectly converted from 0).\n',N); end;
            
            idxRem = cellfun(@(x,y) x | y, idxRem, idxRem2, 'UniformOutput',false);
            
            %=== Now, assume the rest of the values have been erroneously
            % converted from F to C, so convert them back.
            idxManip = cellfun(@(x) x<20, tmp(:,4),'UniformOutput',false);
            low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            low = cellfun(@(x) x*9/5+32, low, 'UniformOutput',false);
            idxManip2 = cellfun(@(x) x<10, low, 'UniformOutput',false);
            for m=1:size(low,1)
                low{m}(idxManip2{m}) = low{m}(idxManip2{m})*9/5+32;
            end
            
            %=== spooky action at a distance
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = low{m};
            end
            
            if verboseFlag; fprintf('Replaced %2.0f values which were incorrectly converted.\n',sum(cell2mat(idxManip))); end;
            if verboseFlag; fprintf('Replaced %2.0f values of these values twice.\n',sum(cell2mat(idxManip2))); end;
        case 'TroponinI'
            % Reference ranges: 0-0.4 µg/L
            % Could be off by orders of 10
            idxManip = cellfun(@(x) (x>30), tmp(:,4),'UniformOutput',false);
            low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            low = cellfun(@(x) x*0.1, low, 'UniformOutput',false);
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = low{m};
            end
            if verboseFlag; fprintf('Multiplied %2.0f values by 0.11 (30->0.3).\n',sum(cell2mat(idxManip))); end;
        case 'TroponinT'
            % Reference ranges: 0-0.1 µg/L
            % Could be off by orders of 10.
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'Urine'
            if verboseFlag; fprintf('No preprocessing performed.\n'); end;
        case 'WBC' % cells/nL
            % Reference range: 4.5-11.0
            % 1 (10^3)/µL	== 1 (10^9)/L
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
            if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
        case 'Weight' % kg
            % Reference range:
            %   86.6 kg (190.9 lb) (males)
            %   74.4 kg (164.0 lb) (females)
            % 1 kilogram = 2.20462262 pounds, 1 pound = 0.45359237 kilograms
            % 1 kilogram = 0.15747304 stones, 1 stone = 6.35029318 kilograms
            
            %=== Impute NaN in the first weight, delete the others
            idxManip = cellfun(@(x) (x==-1), tmp(:,4),'UniformOutput',false);
            low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
            low = cellfun(@(x) NaN, low, 'UniformOutput',false);
            for m=1:size(tmp,1)
                tmp{m,4}(idxManip{m}) = low{m};
            end
            if verboseFlag; fprintf('Replaced %2.0f values which were -1 with NaN.\n',sum(cell2mat(idxManip))); end;
            
            [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq');
            if verboseFlag; fprintf('Deleted %2.0f values which were 0.\n',N); end;
            
            %=== The following values continually pop up:
            % 0.6 is on patient 3889 (subid 142393)
            % Their weight is constant at 70, then becomes 0.6 at 467 min
            [idxRem2,N] = pnPreprocessRemovalIndices(tmp, idx, 35, 'lt');
            idxRem = cellfun(@(x,y) x | y, idxRem, idxRem2, 'UniformOutput',false);
            if verboseFlag; fprintf('Deleted %2.0f values which were less than 35.\n',N); end;
    end
    
    %=== Replace data with tmp
    for m=2:size(data,2)
        data_processed(:,m) = cellfun(@pnPreprocessReplaceData, data_processed(:,m), tmp(:,m), idx, 'UniformOutput', false);
    end
    
    %=== Remove deleted data from 'data'
    if ~isempty(idxRem)
        for m=2:size(data,2)
            data_processed(:,m) = cellfun(@pnPreprocessDeleteData, data_processed(:,m), idxRem, 'UniformOutput', false);
        end
    end
end

end