refactor_plans.js

// set commandline flags defaults

var DEBUG = false;

var SKIP_PRE_2000 = true; // don't save grads from before 2000

var UPDATE_ARCHIVES = true; // update the archive directories (ie reset cmte fsuids in directories containing documents)

var ADD_DERIVED_VALUES = true; // add calcualted values, like time since admission, time since prelim, residency

var DELETE_OLD_FIELDS = false; // get rid of deprecated fields

var SAVE_REFACTORED_PLANS = true; // save the refactored plans as "allstudents.json" or argv[3]

var VERBOSE = false;

var FIXEDDATE = false; // use next argument in form YYYYMMDD as current date

var TODAYSDATE =  new Date();

var input_students_path = __dirname + '/jekyll/_data/allstudents_old_format.json';
var output_students_path = __dirname + '/jekyll/_data/allstudents_lit.json';



// read command line options



for (var i = 0; i < process.argv.length; i++) {

	if ("-i" == process.argv[i] && i < process.argv.length - 1) {
		input_students_path = process.argv[i + 1];
	}

	if ("-o" == process.argv[i] && i < process.argv.length - 1) {
		output_students_path = process.argv[i + 1];
	}

	if (-1 != process.argv[i].indexOf("DEBUG")) {
		DEBUG = process.argv[i].startsWith("-") ? false : true;
	}
	if (-1 != process.argv[i].indexOf("SKIP_PRE_2000")) {
		SKIP_PRE_2000 = process.argv[i].startsWith("-") ? false : true;
	}
	if (-1 != process.argv[i].indexOf("UPDATE_ARCHIVES")) {
		UPDATE_ARCHIVES = process.argv[i].startsWith("-") ? false : true;
	}
	if (-1 != process.argv[i].indexOf("ADD_DERIVED_VALUES")) {
		ADD_DERIVED_VALUES = process.argv[i].startsWith("-") ? false : true;
	}
	if (-1 != process.argv[i].indexOf("DELETE_OLD_FIELDS")) {
		DELETE_OLD_FIELDS = process.argv[i].startsWith("-") ? false : true;
	}
	if (-1 != process.argv[i].indexOf("SAVE_REFACTORED_PLANS")) {
		console.log("Save_factored_plans switch");
		SAVE_REFACTORED_PLANS = process.argv[i].startsWith("-") ? false : true;
	}
	if (-1 != process.argv[i].indexOf("VERBOSE")) {
		console.log("Verbose switch");
		VERBOSE = process.argv[i].startsWith("-") ? false : true;
	}
     if (-1 != process.argv[i].indexOf("FIXEDDATE")) {
        console.log("Date switch");
        FIXEDDATE = process.argv[i].startsWith("-") ? false : true;
        TODAYSDATE= new Date(process.argv[i+1]);
        if (isNaN(TODAYSDATE)) {
           console.log("BAD FIXEDDATE");
           process.exit(1);
        }
     }

}

console.log("refactor_plans");

if (VERBOSE) {
	console.log("input_students_path = " + input_students_path);
	console.log("output_students_path = " + output_students_path);
	console.log("DEBUG = " + DEBUG);
	console.log("SKIP_PRE_2000 = " + SKIP_PRE_2000);
	console.log("UPDATE_ARCHIVES = " + UPDATE_ARCHIVES);
	console.log("ADD_DERIVED_VALUES = " + ADD_DERIVED_VALUES);
	console.log("DELETE_OLD_FIELDS = " + DELETE_OLD_FIELDS);
	console.log("SAVE_REFACTORED_PLANS = " + SAVE_REFACTORED_PLANS);
	console.log("VERBOSE = " + VERBOSE);
}




//----------------------------------------------------------------------------------------

// counters for tracking changes made by this script

var added_ms_plan = 0;
var added_phd_plan = 0;
var members_misspelling = 0;
var added_initial_plan = 0;
var added_plan = 0;
var updated_committee = 0;
var updated_chair = 0;
var updated_cochair = 0;
var updated_univ_rep = 0;
var updated_member = 0;
var removed_awards = 0;
var removed_rotations = 0;
var removed_teaching = 0;
var removed_seminars = 0;
var removed_program_code = 0;
var fixed_current_plan_code = 0;
var moved_current_position = 0;
var moved_GRE = 0;
var complete_bowers = 0;
var admit_bowers = 0;
var degree_awarded = 0;
var reset_admit_term = 0;
var reset_last_term = 0;
var deleted_field = 0;
var removed_invalid_keys = 0;

var current_position_remapped = 0;
var current_position_location_remapped = 0;
var ug_country_remapped = 0;

var deleted_keys = [];


//----------------------------------------------------------------------------------------

// Node modules

var fs = require('fs');
var readYAML = require('read-yaml');
require('./logging')(__filename);

var bc = require("./bc_utilities.js");


//----------------------------------------------------------------------------------------

// Constants

// load constants which are invariant across departments
const key_remappings = bc.readJSONfromAppDir(__dirname,'/plan_key_remappings.json');
const iso3 = bc.readJSONfromAppDir(__dirname,'/jekyll/_data/const/iso3.json');
const lacs = bc.readJSONfromAppDir(__dirname,'/jekyll/_data/const/lacs.json');
const country_names = bc.readJSONfromAppDir(__dirname,'/jekyll/_data/const/names.json');

// for validation of allstudents.json
const student_schema = readYAML.sync(__dirname + '/jekyll/_data/schemata/student_schema.yaml');
var current_keys = [];
for (let i = 0; i < student_schema.length; i++) {
	if (student_schema[i].category != "derived") {
		current_keys.push(student_schema[i].key);
	}
}
// console.log(current_keys);

// load departmental academic plans (varies by each department)
const academic_plans = readYAML.sync(__dirname + '/jekyll/_data/dept/academic_plan_codes.yaml');


//----------------------------------------------------------------------------------------

const fsu_terms = readYAML.sync(__dirname + '/jekyll/_data/univ/fsu-terms.yaml');

let todays_term = "2000_fall";
let today_timestamp = TODAYSDATE.getTime();

for (var term_key in fsu_terms) {

	let term_record = fsu_terms[term_key];
	
	let term_start = Date.parse(term_record.begin_date);
	let term_end =  Date.parse(term_record.end_date);
	
	if (term_start <= today_timestamp && today_timestamp <= term_end ) {
	
		todays_term = term_key;
	}

}

function termkeyWithOffset(term_key,offset) {

	let parts = term_key.split("_");
	let year = parts[0];
	let term_name = parts[1];
	
	for (let off = 0; off < offset; off++) {
		if (term_name == "spring") {
			term_name = "summer";
		}
		else  if (term_name == "summer") {
			term_name = "fall";
		}
		else {
			term_name = "spring";
			year++;
		}
	}

	return (year + "_" + term_name);

}
const current_term = todays_term;
const next_term = termkeyWithOffset(todays_term, 1);
const next_term2 = termkeyWithOffset(todays_term, 2);



// load in registration for current term and future terms


function load_registrations(term) {
	let term_directory = __dirname + "/jekyll/_data/term_data/" + term + "/";
	let registration_path = term_directory + "registrations.json";

	if (fs.existsSync(registration_path)) {
		let reg_string = fs.readFileSync(registration_path);
		return JSON.parse(reg_string);

	}
	else return [];
}

const current_registrations = load_registrations(current_term);
const next_registrations = load_registrations(next_term);
const next2_registrations = load_registrations(next_term2);



// current date info

const date_now = TODAYSDATE;
const month = 1 + date_now.getMonth();
const now_string = date_now.getFullYear() + '-' + month + '-' + date_now.getDate();
const day_now = Date.parse(now_string);
const time_now = day_now / 1000; // time_now is a unix seconds timestamp


//----------------------------------------------------------------------------------------

function currentPlanCode(student) {

	// we converted student.academic_plan_code to an array

	if (Array.isArray( student.academic_plan_code) ) {
			return student.academic_plan_code[0];
	}
	return student.academic_plan_code;


}

//----------------------------------------------------------------------------------------

function correct_plan_code(student) {
	// correct for missing academic_plan_code
	if (bc.notUndefinedAndNotNull(student.current_academic_plan_code)) {
		if (bc.isUndefinedOrIsNull(student.academic_plan_code)) {
			student.academic_plan_code = student.current_academic_plan_code;
			fixed_current_plan_code++;
			console.log("FIXED ACADEMIC PLAN FROM CURR CODE: " + student.last_name + " " + student.fsuid + " acad: " + student.academic_plan_code);
		}
		else {
			if (student.academic_plan_code != student.current_academic_plan_code) {
				console.log("MISMATCHED CURRENT ACADEMIC PLAN CODES: " + student.last_name + " " + student.fsuid + "Curr code: " + student.current_academic_plan_code + " acad: " + student.academic_plan_code);
			}
		}
	}
	if (bc.notUndefinedAndNotNull(student.current_academic_plan)) {
		if (bc.isUndefinedOrIsNull(student.academic_plan_code)) {
			student.academic_plan_code = student.current_academic_plan;
			fixed_current_plan_code++;
			console.log("FIXED ACADEMIC PLAN FROM CURR: " + student.last_name + " " + student.fsuid + " acad: " + student.academic_plan_code);
		}
		else {
			if (student.academic_plan_code != student.current_academic_plan) {
				console.log("MISMATCHED CURRENT ACADEMIC PLAN CODES: " + student.last_name + " " + student.fsuid + "Curr: " + student.current_academic_plan_code + " acad: " + student.academic_plan_code);
			}
		}
	}

     

} // correct_plan_code


//----------------------------------------------------------------------------------------

function confirm_student_has_plan(student) {

     if (bc.isUndefinedOrIsNull(student.academic_plans)) {
     
        if (bc.notUndefinedAndNotNull(student.academic_plan_code)) {
        
               console.log(student.fsuid + ": HAS NO PLANS but HAS PLAN_CODE " + student.academic_plan_code);
               student.academic_plans = { };
               student.academic_plans[student.academic_plan_code] = {};
        }
        else {
        
           console.log(student.fsuid + ": HAS NO PLANS");
           
           // process.exit(1);
        }
     
     }
    else  if (bc.notUndefinedAndNotNull(student.academic_plan_code)) {
     
           if (bc.isUndefinedOrIsNull(student.academic_plans[student.academic_plan_code])) {
               console.log(student.fsuid + ": HAS NO PLAN for CURRENT ACADEMIC PLAN CODE");
               process.exit(1);
          }
     
     }


}
 


//----------------------------------------------------------------------------------------

const new_key = [

	"current_position",
	"current_position_location",
	"current_position_type",
	"current_position_updated",
	"current_url",
	"current_email",
	"current_comment",
	"current_address",
	"current_phone"
];

const old_phd_key = [
	"phd_current_postgrad_position",
	"phd_current_position_location",
	"phd_current_position_type",
	"phd_current_position_updated",
	"phd_current_url",
	"phd_current_email",
	"phd_current_comment",
	"phd_current_address",
	"phd_current_phone"
];

const old_ms_key = [
	"ms_current_postgrad_position",
	"ms_current_position_location",
	"ms_current_position_type",
	"ms_current_position_updated",
	"ms_current_url",
	"ms_current_email",
	"ms_current_comment",
	"ms_current_address",
	"ms_current_phone"
];

var delete_old_fields = function(student) {
	// delete  all the old fields that have been refactored

	if (DELETE_OLD_FIELDS) {

		// compare all keys in student with keys in student_schema

		var student_keys = Object.keys(student);

		for (var i = 0; i < student_keys.length; i++) {

			if (-1 == current_keys.indexOf(student_keys[i])) {
				if ("academic_plans" == student_keys[i]) {

					console.log("******************************");
				}
				if (-1 == deleted_keys.indexOf(student_keys[i])) {
					deleted_keys.push(student_keys[i]);
				}
				delete student[student_keys[i]];
				deleted_field++;

			}

		}

	} // delete old fields

}; // delete old fields



//----------------------------------------------------------------------------------------

function cleanup_student(student) {


	// NOTE: eventually we can remove redundant fields that are predefined for each plan_code (like name, abbr, degree, degree type, degree_product)
	// because these could be looked up in a plan_codes table

	//NOTE: we should validate all term codes YYYY/M to make sure they comply with /1 /6 and /9

	var plan_codes  = Object.keys(student.academic_plans);

//	if (bc.notUndefinedAndNotNull(student.academic_plans)) {
//		plan_codes = Object.keys(student.academic_plans);
//	}
//	else {
//
//		console.log(student.last_name + " has no ACADEMIC PLANS");
//
//	}

	var earliest_admit_term = '3000/9';
	var latest_last_term = '1000/1';

	var spaceString = new String(" ");
	var slashString = new String("/");

	for (var i = 0; i < plan_codes.length; i++) {

		let plan_code = plan_codes[i];

		// make sure student overall admit_term and last_term are earliest and lastest terms across all plans
		// make sure that within a plan, that admit_term is set to admit_bowers and last_term is set to complete_bowers
		// make sure if complete_bowers than degree was awarded and awarded in that term

		// 4/1/2019: use GST terms not Bowers terms...
		if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code].admit_bowers)) {
			if (student.academic_plans[plan_code].admit_term != student.academic_plans[plan_code].admit_bowers) {
				console.log('Student: ' + student.last_name + " mismatch admit: " + student.academic_plans[plan_code].admit_term + " vs. bowers: " + student.academic_plans[plan_code].admit_bowers);
				// student.academic_plans[plan_code].admit_term = student.academic_plans[plan_code].admit_bowers;
				admit_bowers++;
			}
		}
		if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code].complete_bowers)) {
			if (student.academic_plans[plan_code].last_term != student.academic_plans[plan_code].complete_bowers) {
				console.log('Student: ' + student.last_name + " mismatch last: " + student.academic_plans[plan_code].last_term + " vs. bowers: " + student.academic_plans[plan_code].complete_bowers);
				// student.academic_plans[plan_code].last_term = student.academic_plans[plan_code].complete_bowers;
				complete_bowers++;

			}

			if (bc.isUndefinedOrIsNull( student.academic_plans[plan_code].degree_awarded )) {
				student.academic_plans[plan_code].degree_awarded = student.academic_plans[plan_code].degree;
				// student.academic_plans[plan_code].degree_awarded_term = student.academic_plans[plan_code].complete_bowers;
				student.academic_plans[plan_code].outcome = "graduated";
				degree_awarded++;
			}
		}

		if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code]['admit_term'])) {

				// replace space typo found in some students
				student.academic_plans[plan_code].admit_term = student.academic_plans[plan_code].admit_term.replace(spaceString, slashString);

				if (earliest_admit_term > student.academic_plans[plan_code].admit_term) {
					earliest_admit_term = student.academic_plans[plan_code].admit_term;
				}
		}

		if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code]['last_term'])) {

				// replace space typo found in some students
				student.academic_plans[plan_code].last_term = student.academic_plans[plan_code].last_term.replace(spaceString, slashString);

				if (latest_last_term < student.academic_plans[plan_code].last_term) {
					latest_last_term = student.academic_plans[plan_code].last_term;
				}
			
		}

		// remove null awards, rotations, seminars, and teaching (unless they have a valid date)

		if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code]['awards'])) {
			for (let a = student.academic_plans[plan_code].awards.length - 1; a >= 0; a--) {
				if (student.academic_plans[plan_code].awards[a].date == null) {
					student.academic_plans[plan_code].awards.splice(a, 1);
					removed_awards++;

				}
			}
		}

		if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code]['rotations'])) {
			for (let a = student.academic_plans[plan_code].rotations.length - 1; a >= 0; a--) {
				if (student.academic_plans[plan_code].rotations[a].date == null) {
					student.academic_plans[plan_code].rotations.splice(a, 1);
					removed_rotations++;

				}
			}

		}
		if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code]['teaching'])) {
			for (let a = student.academic_plans[plan_code].teaching.length - 1; a >= 0; a--) {
				if (student.academic_plans[plan_code].teaching[a].date == null) {
					student.academic_plans[plan_code].teaching.splice(a, 1);
					removed_teaching++;

				}
			}
		}
		if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code]['seminars'])) {
			for (let a = student.academic_plans[plan_code].seminars.length - 1; a >= 0; a--) {
				if (student.academic_plans[plan_code].seminars[a].date == null) {
					student.academic_plans[plan_code].seminars.splice(a, 1);
					removed_seminars++;
				}
			}
		}



		// clean up the chairs and co-chairs, so that either 1 chair, or 2 co-chairs
		if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code].chair )) {
			if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code].co_chair1) && bc.isUndefinedOrIsNull(student.academic_plans[plan_code].co_chair2)) {
				student.academic_plans[plan_code].co_chair2 = student.academic_plans[plan_code].co_chair1;
				student.academic_plans[plan_code].co_chair1 = student.academic_plans[plan_code].chair;
				student.academic_plans[plan_code].chair = null;
			}
			else if (bc.isUndefinedOrIsNull(student.academic_plans[plan_code].co_chair1) && bc.notUndefinedAndNotNull(student.academic_plans[plan_code].co_chair2)) {
				student.academic_plans[plan_code].co_chair1 = student.academic_plans[plan_code].chair;
				student.academic_plans[plan_code].chair = null;
			}
		}

		if (bc.isUndefinedOrIsNull(student.academic_plans[plan_code].chair)) {
			if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code].co_chair1) && bc.isUndefinedOrIsNull(student.academic_plans[plan_code].co_chair2)) {
				student.academic_plans[plan_code].chair = student.academic_plans[plan_code].co_chair1;
				student.academic_plans[plan_code].co_chair1 = null;
			}
			else if (bc.isUndefinedOrIsNull(student.academic_plans[plan_code].co_chair1) && bc.notUndefinedAndNotNull(student.academic_plans[plan_code].co_chair2)) {
				student.academic_plans[plan_code].chair = student.academic_plans[plan_code].co_chair2;
				student.academic_plans[plan_code].co_chair2 = null;
			}

		}


		// remove rotations from everyone except NEURSCIBPD

		if (plan_code != 'NEURSCIBPD') {

			if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code]['rotations'])) {
				delete student.academic_plans[plan_code].rotations;
			}

		}

		// remove redundant academic_program_code
		if (bc.notUndefinedAndNotNull(student.academic_plans[plan_code]['academic_program_code'])) {

			delete student.academic_plans[plan_code].academic_program_code;
			removed_program_code++;

		}


	} // next plan_code

	// make sure admit and last term are updated from plans
	if (earliest_admit_term != '3000/9') {
		if (student.admit_term != earliest_admit_term) {
			student.admit_term = earliest_admit_term;
			reset_admit_term++;
		}
	}
	if (latest_last_term != '1000/1') {
		// only recent last term for graduated students
		if (bc.notUndefinedAndNotNull( student.academic_plan_code)) {
			if (bc.notUndefinedAndNotNull(student.last_term )) {
				if (student.last_term != latest_last_term) {
					student.last_term = latest_last_term;
					reset_last_term++;
				}
			}
		}
	}

} // cleanup_student




//----------------------------------------------------------------------------------------

function term_code_to_unixtime(term) {

	// note that day of term start dates is approximate for each term

	var year = parseInt(term.substring(0, 4), 10);
	var month = parseInt(term.substring(5), 10);
	var day = 1;
	if (month == 6) {
		month = 5;
		day = 10;
	}
	else if (month == 9) {
		month = 8;
		day = 20;
	}

	var term_time = Date.parse(year + '-' + month + '-' + day);

	return (term_time / 1000);

}


//----------------------------------------------------------------------------------------

function endterm_code_to_unixtime(term) {

	// note that day of term end dates is approximate for each term


	var year = parseInt(term.substring(0, 4), 10);
	var month = parseInt(term.substring(5), 10);
	var day = 1;
	if (month == 1) {
		month = 5;
		day = 5;
	}
	else if (month == 6) {
		month = 8;
		day = 5;
	}
	else if (month == 9) {
		month = 12;
		day = 20;
	}


	var term_time = Date.parse(year + '-' + month + '-' + day);

	return (term_time / 1000);

}



//----------------------------------------------------------------------------------------

// TIME-DEPENDENT DERIVED VALUES

function setAge(student) {
	// get age from date of birth string
	if (bc.isUndefinedOrIsNull( student.date_of_birth )) {
		return;
	}
	var birthfields = student.date_of_birth.split('/');
	if (3 > birthfields.length) {
		return;
	}
	var birth_date = new Date();
	birth_date.setFullYear(parseInt(birthfields[2], 10));
	birth_date.setMonth(parseInt(birthfields[0], 10));
	birth_date.setDate(parseInt(birthfields[1], 10));


	student.age = time_now - birth_date / 1000;
	student.age = student.age / 31557600.0;
	student.age = Math.round(student.age * 10) / 10;

	student.age_at_admission = student.time_entered - birth_date / 1000;
	student.age_at_admission = student.age_at_admission / 31557600.0;
	student.age_at_admission = Math.round(student.age_at_admission * 10) / 10;

	if (student.time_left != null) {
		student.age_at_graduation = student.time_left - birth_date / 1000;
		student.age_at_graduation = student.age_at_graduation / 31557600.0;
		student.age_at_graduation = Math.round(student.age_at_graduation * 10) / 10;
	}
	else {
		student.age_at_graduation = null;
	}

	// round to tenth of year

}


//----------------------------------------------------------------------------------------

function calculate_times(student) {

	// set derived time values:

	// time values:
	// time_entered, time_in_program,  years_in_program (to 0.1 years), program_year
	// time_of_prelims, time_since_prelims, years_since_prelims (to 0.1 years)
	// term_entered (as integer 1,6, or 9)

	student.time_now = time_now;



	if (typeof student.admit_term != 'undefined' && student.admit_term != null) {

		student.time_entered = term_code_to_unixtime(student.admit_term);

		student.term_entered = parseInt(student.admit_term.substring(5), 10);
		student.academic_year_entered = parseInt(student.admit_term.substring(0, 4), 10);
		if (student.term_entered < 9) {
			student.academic_year_entered--;
		}
		// zero padded last two digits of entered year
		let yy = student.academic_year_entered % 100;
		student.academic_year_entered_label = (yy < 10) ? ("0" + yy) : "" + yy;

	}
	else {

		console.log(student.fsuid + ": ============NO ADMIT TERM=============");


	}

	// adjust timeline start to start of academic year of entry
	if (student.term_entered == 9) {
		student.time_line_start = student.time_entered;
	}
	else if (student.term_entered == 1) {
		student.time_line_start = student.time_entered - 10512000;
	}
	else if (student.term_entered == 6) {
		student.time_line_start = student.time_entered - 21024000;
	}



	// if student has graduated or left
	if (typeof student.last_term != 'undefined' && student.last_term != null) {

		student.time_left = endterm_code_to_unixtime(student.last_term);

		student.time_since_left = time_now - student.time_left;
		student.years_since_left = student.time_since_left / 31557600.0;
		// round to tenth of year
		student.years_since_left = Math.round(student.years_since_left * 10) / 10;

		// reckon academic time in program, meaning time relative to start of time line (academic year of entry)

		student.term_left = parseInt(student.last_term.substring(5), 10);
		student.academic_year_left = parseInt(student.last_term.substring(0, 4), 10);
		if (student.term_left < 9) {
			student.academic_year_left--;
		}
		// zero padded last two digits of entered year
		let yy = student.academic_year_left % 100;
		student.academic_year_left_label = (yy < 10) ? ("0" + yy) : "" + yy;



	}
	else {

		student.time_left = null;

	}

	if (student.time_left != null) {
		student.time_in_program = student.time_left - student.time_entered;
		student.academic_time_in_program = student.time_left - student.time_line_start;
	}
	else {
		// student hasn't graduated or left

		// reckon academic time in program, meaning time relative to start of time line (academic year of entry)
		student.time_in_program = time_now - student.time_entered;
		student.academic_time_in_program = time_now - student.time_line_start;


	}

	// get age now, and age at admission/graduation from date of birth string
	setAge(student);

	student.years_in_program = student.time_in_program / 31557600.0;
	// round to tenth of year
	student.years_in_program = Math.round(student.years_in_program * 10) / 10;
	student.program_year = Math.ceil(student.time_in_program / 31557600.0);


	// NOTE: refactored times for timeline inside of plan instead of student

	//  console.log("About to look at academic plans for " + student.last_name + " " + student.fsuid);
	if (bc.notUndefinedAndNotNull( student.academic_plans)) {

		let plan_codes = Object.keys(student.academic_plans);

		for (let i = 0; i < plan_codes.length; i++) {
			calculate_plan_times(student.fsuid,student.academic_plans[plan_codes[i]]);
		}
	}

	// reset some student values from current plan values

	var plan_code = currentPlanCode(student);
	var plan = student.academic_plans[plan_code];
	if (bc.notUndefinedAndNotNull(plan)) {
		student.years_in_plan = plan.years_in_program;
	}
	else {
		student.years_in_plan = student.years_in_program;
	}


	//	if ( student.last_name == "Wilson") {
	//	console.log("start: " + student.admit_term +
	//	" end: " + student.last_term +
	//	" time in years: " + student.years_in_program );
	//	}

}




//----------------------------------------------------------------------------------------

function determine_residency(student) {

	// figure out student.residency status
	// student should have canonical GST residency and citizenship
	// but  there are number of derived fields that are dependent on residency and citizenship




	/*
		residency_code [FLRS, NFLR, RFAL, NRAL] -- from GST
		residency  [ usa, intl]
		lac ["lac", null] lac if intl and country is a lac country
		usa_or_lac [true,false]
		in_state_tuition_rate [true,false] true if FLRS or LC
		citizenship = iso3 code for citizenship country
		country [iso3code] from GST
		country_name string
		minority = [true,false]
		in_state_tuition_rate [true, false] -- true if FLRS or LAC
	*/

	// see _data/univ/residencyValues.json for possible values
	
	/* best test for international is if they have a visa ? */

	if (student.residency_code == 'FLRS' || student.residency_code == 'NFLR') {
		student.residency = "usa";
	}
	else if (student.residency_code == 'NRAL') {
		student.residency = "intl";
	}
	else if (bc.notUndefinedAndNotNull(student.visa_permit_type)) {
	     student.residency = "intl";
	}
	else {
	     student.residency = "usa";
	}
	
	
	/* set lac status */
     student.lac = false;
	if ("intl" == student.residency) {
		if (student.country) {
			if (lacs[student.country]) {
				student.lac = true;
			}
		}
	}
	student.usa_or_lac =  ("usa" == student.residency || student.lac );
	
	/* set citizenship */

	if (student.residency != 'usa') {
		if (student.country) {
			student.citizenship = student.country;
			student.country_name = country_names[student.country];
		}
	}
	else {
		student.citizenship = 'USA';
		student.country_name = "United States";
	}

	// only 3 residency codes are non-resident for out of state tuition
	student.in_state_tuition_rate =
		(student.residency_code == 'NFLR' ||
			student.residency_code == "NRAL" ||
			student.residency_code == "RANF") ? false : true;

	// if ethnicity is BL, HI, NA, or Multiple then minority = true


	student.minority = false;
	student.underrepresented_ethnicity = false;
	student.minority_noncitizen = false;

	if (student.ethnicity == "AMIND" ||
		student.ethnicity == "HISPA" ||
		student.ethnicity == "PACIF" ||
		student.ethnicity == "BLACK" ||
		student.ethnicity == "MULTIPLE") {

		student.underrepresented_ethnicity = true;

		if (student.residency == 'usa') {
			student.minority = true;
		}
		else {
			student.minority_noncitizen = true;
		}

	}



}


//----------------------------------------------------------------------------------------

function calculate_plan_times(fsuid, plan) {

	plan.time_now = time_now;
	if (typeof plan.admit_term != 'undefined' && plan.admit_term != null) {

		plan.time_entered = term_code_to_unixtime(plan.admit_term);

		plan.term_entered = parseInt(plan.admit_term.substring(5), 10);
		plan.academic_year_entered = parseInt(plan.admit_term.substring(0, 4), 10);
		if (plan.term_entered < 9) {
			plan.academic_year_entered--;
		}
		// zero padded last two digits of entered year
		let yy = plan.academic_year_entered % 100;
		plan.academic_year_entered_label = (yy < 10) ? ("0" + yy) : "" + yy;

	}
	else {

		console.log(fsuid + ": ============NO ADMIT TERM for PLAN " + plan.plan_code + "=============");


	}

	// adjust timeline start to start of academic year of entry
	if (plan.term_entered == 9) {
		plan.time_line_start = plan.time_entered;
	}
	else if (plan.term_entered == 1) {
		plan.time_line_start = plan.time_entered - 10512000;
	}
	else if (plan.term_entered == 6) {
		plan.time_line_start = plan.time_entered - 21024000;
	}



	// if student has graduated or left
	if (typeof plan.last_term != 'undefined' && plan.last_term != null) {

		plan.time_left = endterm_code_to_unixtime(plan.last_term);

		plan.time_since_left = time_now - plan.time_left;
		plan.years_since_left = plan.time_since_left / 31557600.0;
		// round to tenth of year
		plan.years_since_left = Math.round(plan.years_since_left * 10) / 10;

		// reckon academic time in program, meaning time relative to start of time line (academic year of entry)

		plan.term_left = parseInt(plan.last_term.substring(5), 10);
		plan.academic_year_left = parseInt(plan.last_term.substring(0, 4), 10);
		if (plan.term_left < 9) {
			plan.academic_year_left--;
		}
		// zero padded last two digits of entered year
		let yy = plan.academic_year_left % 100;
		plan.academic_year_left_label = (yy < 10) ? ("0" + yy) : "" + yy;



	}
	else {

		plan.time_left = null;

	}

	if (plan.time_left != null) {
		plan.time_in_program = plan.time_left - plan.time_entered;
		plan.academic_time_in_program = plan.time_left - plan.time_line_start;
	}
	else {
		// student hasn't graduated or left

		// reckon academic time in program, meaning time relative to start of time line (academic year of entry)
		plan.time_in_program = time_now - plan.time_entered;
		plan.academic_time_in_program = time_now - plan.time_line_start;


	}

	plan.years_in_program = plan.time_in_program / 31557600.0;
	// round to tenth of year
	plan.years_in_program = Math.round(plan.years_in_program * 10) / 10;
	plan.program_year = Math.ceil(plan.time_in_program / 31557600.0);

	// to speed up drawing of time line, we pre-calculate the times of 5 events: committee_established, program_of_studies, prelims, proposal, and defense
	// get time since prelims, but avoid parsing a null value if prelims haven't been passed yet -->
	// convert prelim date (or current date if missing) to YYYY-MM-DD format
	if (plan.prelim_exam_passed_date) {
		plan.time_of_prelims = Date.parse(plan.prelim_exam_passed_date);
		plan.time_of_prelims = plan.time_of_prelims / 1000;
		plan.time_since_prelims = time_now - plan.time_of_prelims;
		plan.years_since_prelims = plan.time_since_prelims / 31557600.0;
		// round to tenth of year
		plan.years_since_prelims = Math.round(plan.years_since_prelims * 10) / 10;
		// console.log("time: " + plan.time_of_prelims + " since: " + plan.time_since_prelims + " years: " + plan.years_since_prelims);
	}
	else {
		plan.time_of_prelims = null;
		plan.years_since_prelims = 0;
	}

	if (plan.candidacy_term) {
		plan.time_of_candidacy = endterm_code_to_unixtime(plan.candidacy_term);
	//  plan.time_of_candidacy = plan.time_of_candidacy; already in unix seconds
		plan.time_since_candidacy = time_now - plan.time_of_candidacy;
		plan.years_since_candidacy = plan.time_since_candidacy / 31557600.0;
		// round to tenth of year
		plan.years_since_candidacy = Math.round(plan.years_since_candidacy * 10) / 10;
		// console.log("time: " + plan.time_of_candidacy + " since: " + plan.time_since_candidacy + " years: " + plan.years_since_candidacy);
	}
	else {
		plan.time_of_candidacy = null;
		plan.years_since_candidacy = 0;
	}


	// get time since committee established

	if (plan.committee_established_date) {
		plan.time_of_committee_established = Date.parse(plan.committee_established_date);
		plan.time_of_committee_established = plan.time_of_committee_established / 1000;
		plan.time_since_committee_established = time_now - plan.time_of_committee_established;
		plan.years_since_committee_established = plan.time_since_committee_established / 31557600.0;
		// round to tenth of year
		plan.years_since_committee_established = Math.round(plan.years_since_committee_established * 10) / 10;
	}
	else {
		plan.time_of_committee_established = null;
		plan.years_since_committee_established = 0;
	}

	// get time since program of studies filed

	if (plan.program_of_studies_approved_date) {
		plan.time_of_studies = Date.parse(plan.program_of_studies_approved_date);
		plan.time_of_studies = plan.time_of_studies / 1000;
		plan.time_since_studies = time_now - plan.time_of_studies;
		plan.years_since_studies = plan.time_since_studies / 31557600.0;
		// round to tenth of year
		plan.years_since_studies = Math.round(plan.years_since_studies * 10) / 10;
	}
	else {
		plan.time_of_studies = null;
		plan.years_since_studies = 0;
	}


	// get time since proposal

	if (plan.proposal_defense_passed_date) {
		plan.time_of_proposal = Date.parse(plan.proposal_defense_passed_date);
		plan.time_of_proposal = plan.time_of_proposal / 1000;
		plan.time_since_proposal = time_now - plan.time_of_proposal;
		plan.years_since_proposal = plan.time_since_proposal / 31557600.0;
		// round to tenth of year
		plan.years_since_proposal = Math.round(plan.years_since_proposal * 10) / 10;
	}
	else {
		plan.time_of_proposal = null;
		plan.years_since_proposal = 0;
	}

	// get time since defense_date
	if (plan.defense_date) {
		plan.time_of_defense = Date.parse(plan.defense_date);
		plan.time_of_defense = plan.time_of_defense / 1000;
		plan.time_since_defense = time_now - plan.time_of_defense;
		plan.years_since_defense = plan.time_since_defense / 31557600.0;
		// round to tenth of year
		plan.years_since_defense = Math.round(plan.years_since_defense * 10) / 10;
	}
	else {
		plan.time_of_defense = null;
		plan.years_since_defense = 0;
	}

	// console.log("start: " + plan.admit_term + " end: " + plan.last_term + " time in years: " + plan.years_in_program );

} // calculate_plan_times

//----------------------------------------------------------------------------------------

function update_archive_directories(student) {

	// make sure that there is a ./jekyll/archive/<fsuid>/ directory to store pdf documents,
	// and that it has an updated .cassowary_users file containing the student fsuid and the committee fsuids
	var archive_directory_path = __dirname + '/jekyll/archive';
	if (!fs.existsSync(archive_directory_path)) {
		fs.mkdirSync(archive_directory_path);
	}
	var archive_path = __dirname + '/jekyll/archive/' + student.fsuid;

	if (!fs.existsSync(archive_path)) {
		fs.mkdirSync(archive_path);
	}

	var cassowary_file = archive_path + "/.cassowary_users";

	var cmteIDs = arrayOfCommitteeFSUIDs(student);
	cmteIDs.push(student.fsuid);

	var archive_users = cmteIDs.join(" ");

	fs.writeFile(cassowary_file, archive_users, function(err) {
		if (err) {
			return console.log("Error Writing Cassoway Users for " + student.fsuid + ":" + err);
		}
	});


}


//----------------------------------------------------------------------------------------

function arrayOfCommitteeFSUIDs(student) {

	var cmteIDs = new Array;

//     console.log(student.fsuid +": arrayOfCommitteeFSUIDs");
//      console.log(student.academic_plans);
      
      
     
     if (bc.notUndefinedAndNotNull(student.academic_plan_code)){
          let plan = student.academic_plans[student.academic_plan_code];
          
          if (bc.notUndefinedAndNotNull(plan.chair)) {
               cmteIDs.push(plan.chair.fsuid);
          }
          if (bc.notUndefinedAndNotNull(plan.co_chair1)) {
               cmteIDs.push(plan.co_chair1.fsuid);
          }
          if (bc.notUndefinedAndNotNull(plan.co_chair2)) {
               cmteIDs.push(plan.co_chair2.fsuid);
          }
          if (bc.notUndefinedAndNotNull(plan.univ_rep)) {
               cmteIDs.push(plan.univ_rep.fsuid);
          }
          if (bc.notUndefinedAndNotNull(plan.members)) {
          
               let start_length = plan.members.length;
                bc.removeInvalidKeysFromObject(plan.members);
                
                let end_length = plan.members.length;
                
                if (start_length != end_length) {
                
                    console.log(student.fsuid + " Committee Members cleaned");
                }
                
               for (var i = 0; i < plan.members.length; i++) {
               
               if (bc.isUndefinedOrIsNull(plan.members[i])) {
                     console.log(plan.members);
                    
               }
               
                    if (bc.isUndefinedOrIsNull(plan.members[i].fsuid)) {
                    
                         console.log(student.fsuid + " Committee Member Missing fsuid");
                         console.log(plan.members);
                        // process.exit(1);
                    }
                    else {
                         cmteIDs.push(plan.members[i].fsuid);
                    }
               }
          }
     }


	return cmteIDs;


}

//----------------------------------------------------------------------------------------

function set_disability(student) {

	if (bc.isUndefinedOrIsNull(student.disability) ) {

		student.disability = "no";
	}


} // set_disability

//----------------------------------------------------------------------------------------


//----------------------------------------------------------------------------------------

function set_name(student) {

	student.name = student.last_name + ", " + student.first_name;


} // set name


//----------------------------------------------------------------------------------------

function set_missing_image(student) {

	// TODO: add privacy feature?

	if (bc.isUndefinedOrIsNull(student.image_url)) {

		if (bc.notUndefinedAndNotNull(student.idcard_image_url)) {
			student.image_url = student.idcard_image_url;
		}
	}
}


//----------------------------------------------------------------------------------------

function remap_old_fields(student) {

	if (bc.notUndefinedAndNotNull(student.current_position)) {

		if (bc.notUndefinedAndNotNull ( student.current_postgrad_position) ) {
			student.current_postgrad_position = student.current_postgrad_position + " (" + student.current_position + ")";
		}
		else {
			student.current_postgrad_position = student.current_position;
		}
		current_position_remapped++;

	}

	if (bc.notUndefinedAndNotNull(student.current_position_location) ) {

		if (bc.isUndefinedOrIsNull (student.current_postgrad_location)) {
			student.current_postgrad_location =  student.current_position_location;
		}
		else {
			student.current_postgrad_location = student.current_position_location;
		}
		current_position_location_remapped++;

	}
	if (bc.notUndefinedAndNotNull (student.ug_country )) {
		if (bc.isUndefinedOrIsNull( student.undergrad_country)) {
			student.undergrad_country = student.ug_country;
		}
		ug_country_remapped++;

	}

} // remap_old_fields


function fix_degree_awarded_outcome_and_thesisproposal(student) {



		var plan_keys = Object.keys(student.academic_plans);

		for (var i = 0; i < plan_keys.length; i++) {

			var plan_code = plan_keys[i];
			var plan = student.academic_plans[plan_code];

			if (bc.notUndefinedAndNotNull(plan.degree_awarded)) {
				plan.outcome = "graduated";
			}


			if (bc.notUndefinedAndNotNull(plan.thesis_proposal_defense_passed_date)) {

				plan.proposal_defense_passed_date = plan.thesis_proposal_defense_passed_date;
				delete plan.thesis_proposal_defense_passed_date;
			}


		}
	
}


//----------------------------------------------------------------------------------------

function update_degrees_awarded(student) {


	var keys = Object.keys(student.academic_plans);
	for (var i = 0; i < keys.length; i++) {
		var plan_code = keys[i];

		if (student.academic_plans[plan_code].outcome == 'graduated') {
			if (bc.notUndefinedAndNotNull( academic_plans[plan_code])) {
				let degree = academic_plans[plan_code].degree;

				if (typeof student.degrees == 'undefined' || student.degrees == null ||
					student.degrees.length == 0) {
					student.degrees = degree;
				}
				else {
					if (-1 == student.degrees.indexOf(degree)) {
						student.degrees = student.degrees + ", " + degree;
					}
				}
			}

		}

	}


}

//----------------------------------------------------------------------------------------

function update_registration_flags(student) {

	// TODO:  put all registration into map, so we can look backwards
	// 

	// TODO: figure out how to set prelim flag true if prelims have been taken

	// registered_for_BSC8985
	// registered_for_bsc8964
	// registered_for_BSC8976	
	
	
		

     function termkey2code(termkey) {
          console.log(termkey);
          let parts = termkey.split("_");
          let month = 1;
          if ("summer" == parts[1]) {
               month = 6;
          }
          if ("fall" == parts[1]) {
               month = 9;
          }

          return (parts[0] + "/" + month);

     }
		

	// must have an active plan
	let plan_code = student.academic_plan_code;
	if (bc.isUndefinedOrIsNull(plan_code)) {
		return;
	}


	let registration_requirements = [];


	for (let i in academic_plans[plan_code].requirements) {

		let requirement = academic_plans[plan_code].requirements[i];

		if (requirement.value_type == "registration") {
			registration_requirements.push(requirement);
		}
	}

     function check_registration(registrations,course_code) {
          return registrations.reduce(function(accumulator, item) {

               if (item.emplid == student.emplid) {
                    accumulator = accumulator || item.classes.reduce(function(acc, class_item) {
                         acc = acc || class_item.class_code.includes(course_code);
                         return acc;
                    }, false);

               }
               return accumulator;
          }, false);
     }


// TODO: check all terms for all registrations, not just adjacent terms, for historical update

	for (let i in registration_requirements) {

		let course_code = registration_requirements[i].course_value;
		let key = registration_requirements[i].key;


		//  if (typeof  student.academic_plans[plan_code][key] != "undefined" &&  student.academic_plans[plan_code][key] != null ) {
		// preserve pre-existing flags

		student.academic_plans[plan_code][key] = check_registration(current_registrations,course_code) ? termkey2code(current_term) : student.academic_plans[plan_code][key];
		student.academic_plans[plan_code][key] = check_registration(next_registrations,course_code) ? termkey2code(next_term) : student.academic_plans[plan_code][key];
		student.academic_plans[plan_code][key] = check_registration(next2_registrations,course_code) ? termkey2code(next_term2) : student.academic_plans[plan_code][key];

		// if (  student.academic_plans[plan_code][key]) {
		//					console.log(student.fsuid + " " + key + " registered");
		//				}
		// }

	}

}


//----------------------------------------------------------------------------------------

var _removeInvalidKeysFromStudent = function(student) {

     let old_student =  JSON.stringify(student,null,"    ");

     bc.removeInvalidKeysFromObject(student);

     let new_student = JSON.stringify(  student,null,"    ");

     if (old_student != new_student) {
          removed_invalid_keys++;
     }

};


//----------------------------------------------------------------------------------------

var refactor = function(students) {

	var num_students = students.length;

	console.log("Num: " + num_students);

	for (let i = num_students - 1; i >= 0; i--) {
	
		// get admit time in months
		// don't copy over students admitted prior to fall 2000

		if (typeof students[i].admit_term != 'undefined' || students[i].admit_term != null) {
		
			let parts = students[i].admit_term.split("/");
			var admit_year = parseInt(parts[0],10);
			var admit_month = parseInt(parts[1],10);
		}
		else {

			admit_month = 0;
			admit_year = 0;
			console.log(" MISSING ADMIT TERM: " + students[i].fsuid);

		}
		let admit_time = (admit_month * 1) + (admit_year * 12);


		// 24081 = fall 2006
		// 24009 = fall 2000
		// 23985 = fall 1998 (when houpt arrived)
		if (admit_time < 23985 && SKIP_PRE_2000) {
			students.splice(i, 1);
		}
		else {


			// a few left over fields that need remapping for mongo-sourced data
			// should not be required for GST-sourced data
			
			correct_plan_code(students[i]);
			
			confirm_student_has_plan(students[i]);			
			
			remap_old_fields(students[i]);
			delete_old_fields(students[i]);
			cleanup_student(students[i]);
			fix_degree_awarded_outcome_and_thesisproposal(students[i]);
			

			// these apply to all data, whether mongo or GST-sourced data
			set_disability(students[i]); // set disability to "no" by default
			set_missing_image(students[i]);
			update_degrees_awarded(students[i]);


			// get rid of nulls or empty fields
			_removeInvalidKeysFromStudent(students[i]);
			
			

			// These changes applied after every download from mongo
			// to update things and calculate derived times, residency, and name

			if (UPDATE_ARCHIVES) {

				update_archive_directories(students[i]);

			}
			
			if (ADD_DERIVED_VALUES) {
			
				update_registration_flags(students[i]);
				calculate_times(students[i]);
				determine_residency(students[i]);
				set_name(students[i]);

			}

			// get rid of nulls or empty fields
			_removeInvalidKeysFromStudent(students[i]);
	

		}

	}

	num_students = students.length;


	console.log("Reduced Num: " + num_students);

	if (VERBOSE) {
		console.log("added_ms_plan: " + added_ms_plan);
		console.log("added_phd_plan: " + added_phd_plan);
		console.log("members_misspelling: " + members_misspelling);
		console.log("added_initial_plan: " + added_initial_plan);
		console.log("added_plan: " + added_plan);
		console.log("updated_committee: " + updated_committee);
		console.log("updated_chair: " + updated_chair);
		console.log("updated_cochair: " + updated_cochair);
		console.log("updated_univ_rep: " + updated_univ_rep);
		console.log("updated_member: " + updated_member);
		console.log("removed_awards: " + removed_awards);
		console.log("removed_rotations: " + removed_rotations);
		console.log("removed_teaching: " + removed_teaching);
		console.log("removed_seminars: " + removed_seminars);
		console.log("removed_program_code: " + removed_program_code);
		console.log("fixed_current_plan_code: " + fixed_current_plan_code);
		console.log("moved_current_position: " + moved_current_position);
		console.log("moved_GRE: " + moved_GRE);
		console.log("complete_bowers: " + complete_bowers);
		console.log("admit_bowers: " + admit_bowers);
		console.log("degree_awarded: " + degree_awarded);
		console.log("reset_admit_term: " + reset_admit_term);
		console.log("reset_last_term: " + reset_last_term);

		console.log("current_position_remapped: " + current_position_remapped);
		console.log("current_position_location_remapped: " + current_position_remapped);
		console.log("ug_country_remapped: " + ug_country_remapped);

		console.log("deleted_field: " + deleted_field);

		console.log("removed_invalid_keys: " + removed_invalid_keys);

		for (let i = 0; i < deleted_keys.length; i++) {
			console.log("	" + deleted_keys[i]);
		}

	} // end verbose


	
}; // refactor


//----------------------------------------------------------------------------------------

function save(students) {

	if (SAVE_REFACTORED_PLANS) {
		bc.backupFile(output_students_path);
		fs.writeFile(output_students_path, JSON.stringify(students, null, "\t"), function(err) {
			if (err) {
				return console.log(err);
			}
			else {
				// make backup copy of this refactored allstudents.json
				bc.backupFile(output_students_path);
			}
		});
	}

}


//----------------------------------------------------------------------------------------