gst-to-gradphile

// -------------------------------------------------------------------------
// -------------------------------------------------------------------------
// -------------------------------------------------------------------------

if (typeof process.argv[2] == "undefined") {
	console.log("\n");
	console.log("You must specify the academic organization, eg ASBIO, as a command line argument");
	console.log("USAGE: node " + process.argv[1] + " <ACADORG>  ['update'] [FIXEDDATE MM/DD/YYYY]");
	process.exit(1);
	console.log("\n");
}
var org_abbr = process.argv[2];

var updating = false;

var FIXEDDATE = false;
var TODAYSDATE= new Date();

var faculty_path = __dirname + '/../source_data/dept/faculty.csv';

var output_students_path = __dirname+'/' + org_abbr + '_allstudents_old_format.json';
var pre_output_students_path =  __dirname+'/' + org_abbr + '_allstudents_old_format_preupdate.json';
var input_students_path = __dirname + '/../source_data/students/allstudents_old_format.json';

// read command line options



for (let i = 0; i < process.argv.length; i++) {

	if ("-i" == process.argv[i] && i < process.argv.length - 1) {
		input_students_path = process.argv[i + 1];
	}

	if ("-o" == process.argv[i] && i < process.argv.length - 1) {
		output_students_path = process.argv[i + 1];
	}

	if (-1 != process.argv[i].indexOf("update")) {
		updating = process.argv[i].startsWith("-") ? false : true;
	}

     if (-1 != process.argv[i].indexOf("FIXEDDATE")) {
        console.log("Date switch");
        FIXEDDATE = process.argv[i].startsWith("-") ? false : true;
        TODAYSDATE= new Date(process.argv[i+1]);
        if (isNaN(TODAYSDATE)) {
           console.log("BAD FIXEDDATE");
           process.exit(1);
        }
     }

}

const fs = require('fs');
const bc = require('../bc_utilities.js');
require('../logging')(__filename);

// login needed to retrieve ldap_info

var username_regex = new RegExp('FSUID=(.+)\n', 'gm');

var password_regex = new RegExp('PASSWORD=(.+)[\n$]', 'gm');

var secrets = fs.readFileSync(__dirname + "/GST_SECRETS.sh").toString('utf8');

// console.log(secrets);

const password = password_regex.exec(secrets)[1];
const username = username_regex.exec(secrets)[1];

var date_now = TODAYSDATE;
var todays_date = (date_now.getMonth() + 1) + "/" + date_now.getDate() + "/" + date_now.getFullYear();
var todays_date_string = date_now.toString();

function fourdigit2termcode(fourdigit) {

	// fourdigit code leaves out century digit: 2019/9 -> 2199

	if (0 == fourdigit.length) {
		return null;
	}

	var month = fourdigit % 10;
	var year_code = Math.floor(fourdigit / 10);
	var year = Math.floor(year_code / 100) * 1000 + year_code % 100;
	if (year < 2000) {
		year += 900;
	}

	//  console.log(fourdigit + " --> " + year.toString() + '/' + month.toString());
	return year.toString() + '/' + month.toString();

}

function dayMonthYear2termcode(dayMonthYear) {

	// daymonthyear dd-mm-YYYY

	var month2Int = {

		"JAN": 1,
		"FEB": 2,
		"MAR": 3,
		"APR": 4,
		"MAY": 5,
		"JUN": 6,
		"JUL": 7,
		"AUG": 8,
		"SEP": 9,
		"OCT": 10,
		"NOV": 11,
		"DEC": 12

	};

	if (0 == dayMonthYear.length) {
		return null;
	}
	var fields = dayMonthYear.split("-");

	var date = parseInt(fields[0], 10);
	var month = month2Int[fields[1]];
	var year = parseInt(fields[2], 10) + 2000;

	var term = 1;

	if ((month == 8 && date > 25) || (month >= 9)) {
		term = 9;
	}
	else if (month >= 5) {
		term = 6;
	}

	//  console.log(fourdigit + " --> " + year.toString() + '/' + month.toString());
	return year.toString() + '/' + term.toString();

}

function termcode2months(termcode) {
	let comp = termcode.split('/');
	return parseInt(comp[0], 10) * 12 + parseInt(comp[1], 10);
}

function match_student_with_gst_record_by_emplid(students, gst_record) {

	// return a matching student, or create a new one and add it to students array

	let record_id = parseInt(gst_record["EMPLID"], 10);
	for (let i = 0; i < students.length; i++) {
		var student = students[i];
		//console.log("compare student " + students["emplid"] + " record " + record_id);
		if (student["emplid"] == record_id) {
			return students[i];
		}
	}

	return null;

} // match_student_with_gst_record_by_emplid

function match_plan_with_gst_record_by_trackid(student, gst_record) {

	// return a matching student, or create a new one and add it to students array

	let track_id = parseInt(gst_record["STUDENTTRACKID"], 10);
	var plan_keys = Object.keys(student['academic_plans']);
	for (let i = 0; i < plan_keys.length; i++) {
		var plan_code = plan_keys[i];
		//console.log("compare student " + students["emplid"] + " record " + record_id);
		if (student['academic_plans'][plan_code]["gst_trackid"] == track_id) {
			return plan_code;
		}
	}

	return null;

} // match_plan_with_gst_record_by_trackid

// var gst_tracks = require(__dirname+'/'+org_abbr+'_gst_active_tracks_extended.json');
// update_active_students_from_gst_tracks(allstudents,gst_tracks);

var gst_plans = bc.readJSONfromAppDir(__dirname,'/'+org_abbr+'_gst_student_plans_extended.json');
var gst_scores = bc.readJSONfromAppDir(__dirname,'/'+org_abbr+'_gst_testscores.json');
var gst_cmtes = bc.readJSONfromAppDir(__dirname,'/'+org_abbr+'_gst_cmtes.json');
var gst_objects = bc.readJSONfromAppDir(__dirname,'/'+org_abbr+'_gst_student_objects.json');

function updatedFromGST(student,key) {

// keep track if student updated, and keep an array of updated keys,  in a map accessed by today's date

     
     if (bc.isUndefinedOrIsNull(student.updated_gst_keys)) {
          student.updated_gst_keys = {};
          
             
     }
     if (bc.isUndefinedOrIsNull( student.updated_gst_keys[todays_date_string])) {
          student['_updated_from_gst'] = true;
           student['_last_updated_from_gst'] = todays_date_string;
           student.updated_gst_keys[todays_date_string] = [];
    }  
     
     student.updated_gst_keys[todays_date_string].push(key);

}

function update_active_students_from_gst_tracks(students, gst_tracks) {

	for (var i = 0; i < gst_tracks.length; i++) {
		var gst_record = gst_tracks[i];
		let student = match_student_with_gst_record_by_emplid(students, gst_record);
		if (null == student) {
			// create a new student
			student = {};
			student["emplid"] = parseInt(gst_record["EMPLID"], 10);
			student["academic_plans"] = {};

			students.push(student);
		}

		update_student_from_gst_record(student, gst_record);
		// console.log(student);
	} // next gst_record

} // update_active_students_from_gst_tracks

function update_student_from_gst_record(student, gst_record) {

	// PERSONAL INFO
	// overwrite name, email, and address etc from gst
	student["fsuid"] = gst_record["FSUID"].toLowerCase();
	student['full_name'] = gst_record["STUDENTNAME"].trim();

	var name_parts = student['full_name'].split(",");
	name_parts[1] = name_parts[1].trim();
	var first_name_parts = name_parts[1].split(" ");
	if (bc.isUndefinedOrIsNull(student['first_name'])) {
		student['first_name'] = first_name_parts[0];
	}
	if (bc.isUndefinedOrIsNull(student['last_name'])) {
		student['last_name'] = name_parts[0].trim();
	}
	student["campus_email"] = gst_record["CAMP_EMAIL"]; //*****
	student["preferred_email"] = gst_record["PREF_EMAIL"]; //*****
	// TODO: make sure students know that they can put dept email as their preferred email
	if (org_abbr != "ASBIO") {
		student["email"] = gst_record["PREF_EMAIL"];
	}

	var address_lines = new Array;
	if (gst_record["MAIL_ADDRESS1"].length > 0) {
		address_lines.push(gst_record["MAIL_ADDRESS1"]);
	}
	if (gst_record["MAIL_ADDRESS2"].length > 0) {
		address_lines.push(gst_record["MAIL_ADDRESS2"]);
	}
	if (gst_record["MAIL_CITY"].length > 0) {
		address_lines.push(gst_record["MAIL_CITY"]);
	}
	if (gst_record["MAIL_STATE"].length > 0) {
		address_lines.push(gst_record["MAIL_STATE"]);
	}
	if (gst_record["MAIL_POSTAL"].length > 0) {
		address_lines.push(gst_record["MAIL_POSTAL"]);
	}

	student["address"] = "";
	for (var i = 0; i < address_lines.length; i++) {
		student["address"] = student["address"] + address_lines[i];
		if (i < address_lines.length - 1) {
			student["address"] = student["address"] + ", ";
		}
	}

	student["idcard_image_url"] = "https://images.its.fsu.edu/fsucardimages/" + gst_record["IMAGE_NAME"];
	if (typeof student["image_url"] == undefined || student["image_url"] == null) {
		student["image_url"] = student["idcard_image_url"];
	}
	student["visa_permit_type"] = gst_record["VISA_PERMIT_TYPE"];
	student["date_of_birth"] = gst_record["BIRTHDAY"];
	student["country"] = gst_record["COUNTRY"];

	if (bc.notUndefinedAndNotNull(gst_record["GENDER"])) {
		student["sex"] = gst_record["GENDER"];
	}
	if (bc.notUndefinedAndNotNull(gst_record["RESIDENCY"])) {
		student["residency_code"] = gst_record["RESIDENCY"];
	}
	if (bc.notUndefinedAndNotNull(gst_record["ETHNIC_GRP_CD"])) {
		student["ethnicity"] = gst_record["ETHNIC_GRP_CD"];
	}

} // update_student_from_gst_record

function match_student_plan_with_gst_plan_code(student, gst_plan_code) {
	if (Object.keys(student.academic_plans).includes(gst_plan_code)) {
		return student.academic_plans[gst_plan_code];
	}
	return null;
} // match_student_plan_with_gst_plan_code

function update_students_from_gst_plans(students, gst_plans) {

	for (var i = 0; i < gst_plans.length; i++) {
		var gst_plan = gst_plans[i];
		let student = match_student_with_gst_record_by_emplid(students, gst_plan);

		if (null == student) {
			// create a new student
			student = {};
			student["emplid"] = parseInt(gst_plan["EMPLID"], 10);
			student["academic_plans"] = {};
			students.push(student);
		}

		// TODO: add initial plan code (or fake it?)

		update_student_from_gst_record(student, gst_plan);
		update_plan_from_gst_plan(student, gst_plan);
		//console.log(gst_plan);
		//console.log(student);

	} // next gst_record

} // update_plans_from_gst_plans

function update_plan_from_gst_plan(student, gst_plan) {

	// need all actve fields for past students too

	var gst_plan_code = gst_plan["LAST_ACAD_PLAN"];

	var plan = match_student_plan_with_gst_plan_code(student, gst_plan_code);

	if (null == plan) {
		//	console.log("PLAN MISMATCH " + student.last_name + " " + student.fsuid + " " + gst_plan_code);
		plan = {};
		student.academic_plans[gst_plan_code] = plan;
	}

	// console.log('about to get admit_term: ' + gst_plan["ADMIT_TERM"]);
	let admit_term = fourdigit2termcode(gst_plan["ADMIT_TERM"]);
	//	console.log('about to get last_term: ' + gst_plan["COMPLETION_TERM"]);

	let last_term = null; // last_term will be null if  status is "Discontinued" or "Deceased"
	
	
	/* 
	     PROG_STATUS_TEXT options:
	          "Active in Program"
	          "Leave of Absence"
	          "Completed Program"
	          "Deceased"
	          "Dismissed" ? haven't seen
	     
	*/

     if (gst_plan["PROG_STATUS_TEXT"] == "Completed Program") {
     
          last_term = fourdigit2termcode(gst_plan["COMPLETION_TERM"]);
     }
     else if (gst_plan["PROG_STATUS_TEXT"] == "Deceased") {
     
     	  if (bc.notUndefinedAndNotNull(gst_plan["DISCONTINUED"])) {
          	last_term = dayMonthYear2termcode(gst_plan["DISCONTINUED"]);
          }
          // but discontinued may not be entered for deceased
          if (null == last_term) {
               last_term = "2014/9"; // for Yize Shao fsuid ys11b
          }
     
     }
	else if (gst_plan["PROG_STATUS_TEXT"] != "Active in Program" && gst_plan["PROG_STATUS_TEXT"] != "Leave of Absence") {
     // catch everything else?
		if (null == last_term && bc.notUndefinedAndNotNull(gst_plan["DISCONTINUED"])) {
			// TODO: current GST doesn't give a completion term unless the student completed,
			// so we have to fake for did not finish students...
			// 3/2018: Yancey extended to "Discontinued" students,
			// but not for dismissed or deceased?
			last_term = dayMonthYear2termcode(gst_plan["DISCONTINUED"]);
			//console.log(gst_plan["STUDENTNAME"] + " discontinued: " + gst_plan["DISCONTINUED"] + " -> " + last_term);
		}

	}

	if (("DISCONTINUED" in gst_plan) && gst_plan["DISCONTINUED"].length > 0) {
		plan["discontinued"] = dayMonthYear2termcode(gst_plan["DISCONTINUED"]);
	}
	else {
		plan["discontinued"] = null;
	}

// NOTE: we don't trust GST admit terms, because they set it to time of admission after a major plan change?
// but we trust GST last terms, because they know when student officially graduated/left
	plan["gst_admit_term"] = admit_term;
	plan["gst_last_term"] = last_term;
	
	
	if (bc.isUndefinedOrIsNull(plan["admit_term"])) {
		plan["admit_term"] = admit_term;
	}
	else if (plan["admit_term"] != admit_term) {
		plan["admit_term_mismatch"] = plan["admit_term"] + " <-> gst: " + admit_term;
	}
	
	
	if (bc.isUndefinedOrIsNull(plan["last_term"])) {
		plan["last_term"] = last_term;
	} 
	else	if (plan["last_term"] != last_term) {
		plan["last_term_mismatch"] = plan["last_term"] + " <-> gst: " + last_term;
		plan["last_term"] = last_term;
	}

	if (last_term != null) {
		// TODO: translate gst status to gradphile outcomes, or vice versa

		// TODO: check if student just switched plans, so -> transferred

		var gst_status_to_gradphile_outcome = {

			"Completed Program": "graduated",
			"Discontinued": "did not finish",
			"Dismissed": "did not finish",
			"Deceased": "did not finish"

		};

		if (null != gst_status_to_gradphile_outcome[gst_plan["PROG_STATUS_TEXT"]]) {
			plan['outcome'] = gst_status_to_gradphile_outcome[gst_plan["PROG_STATUS_TEXT"]];
			plan['outcome_comment'] = gst_plan["PROG_STATUS_TEXT"];
		}
	}

	plan["status"] = gst_plan["PROG_STATUS_TEXT"]; //*****
	plan["campus"] = gst_plan["CAMPUS"]; //*****
	plan["academic_career"] = gst_plan["ACAD_CAREER"]; //*****
	plan["academic_career_text"] = gst_plan["ACAD_CAREER_TEXT"]; //*****
	plan["career_number"] = gst_plan["STDNT_CAR_NBR"]; //*****

	plan["academic_subplans"] = new Array; //*****
	if (gst_plan["LAST_ACAD_SUBPLAN"].length > 0) {
		let subplan = {};
		subplan["academic_subplan"] = gst_plan["LAST_ACAD_SUBPLAN"]; //*****
		subplan["academic_subplan_name"] = gst_plan["ACAD_SUBPLAN_TEXT"]; //*****
		plan["academic_subplans"].push(subplan);
	}
	if (gst_plan["LAST_ACAD_SUBPLAN2"].length > 0) {
		let subplan = {};
		subplan["academic_subplan"] = gst_plan["LAST_ACAD_SUBPLAN2"]; //*****
		subplan["academic_subplan_name"] = gst_plan["ACAD_SUBPLAN2_TEXT"]; //*****
		plan["academic_subplans"].push(subplan);
	}
	if (gst_plan["LAST_ACAD_SUBPLAN3"].length > 0) {
		let subplan = {};
		subplan["academic_subplan"] = gst_plan["LAST_ACAD_SUBPLAN3"]; //*****
		subplan["academic_subplan_name"] = gst_plan["LAST_ACAD_SUBPLAN3_TEXT"]; //*****
		plan["academic_subplans"].push(subplan);
	}
	if (plan["academic_subplans"].length == 0) {
		plan["academic_subplans"] = null;
	}

	plan["degree_checkout_status"] = gst_plan["CHKOUT_STAT_TEXT"]; //*****

	plan["gst_trackid"] = gst_plan["STUDENTTRACKID"]; //*****
	plan["academic_program"] = gst_plan["LAST_ACAD_PROG"]; //*****
	plan["academic_program_text"] = gst_plan["ACAD_PROG_TEXT"]; //*****

	if (gst_plan["DOCTORALCANDIDATE"] == "Y") {
		plan["candidacy_term"] = gst_plan["ADMCANDIDACYYEAR"] + "/" + parseInt(gst_plan["ADMCANDIDACYMONTH"],10);
	}

	// TODO: run through plan and null out all zero-length fields

} // update_plan_from_gst_plan

function update_committees_from_gst_cmtes(students, gst_cmtes) {
	for (var trackid in gst_cmtes) {
		var gst_cmte = gst_cmtes[trackid];
		var gst_record = {};
		gst_record["EMPLID"] = gst_cmtes[trackid][0]['EMPLID_STU'];

		// console.log(gst_cmtes[trackid]);
		// console.log("EMPLID_STU: " + gst_cmtes[trackid][0]['EMPLID_STU']);

		// console.log("CMTE Trackid: " + trackid + " emplid: " + gst_record["EMPLID"]);

		var student = match_student_with_gst_record_by_emplid(students, gst_record);

		// console.log("CMTE Trackid: " + trackid + " emplid: " + gst_record["EMPLID"] + " " + student.last_name)

		if (null != student) {
			update_student_from_gst_cmte(student, trackid, gst_cmte);
		}

	} // next gst_record

} // update_committees_from_gst_cmtes

function update_student_from_gst_cmte(student, trackid, gst_cmte) {

	for (let plan_code in student.academic_plans) {
		var plan = student.academic_plans[plan_code];
		// console.log(plan["gst_trackid"] + '==' + gst_cmte[0]["STUDENTTRACKID"]);

		if (plan["gst_trackid"] == gst_cmte[0]["STUDENTTRACKID"]) {

			if (gst_cmte.length > 0) {

				// overwrite pre-exisiting local committees
				plan["chair"] = null;
				plan["co_chair1"] = null;
				plan["co_chair2"] = null;
				plan["univ_rep"] = null;
				plan["members"] = [];

				for (var i = 0; i < gst_cmte.length; i++) {

					let gst_member = gst_cmte[i];

					let member = {};
					member["name"] = gst_member["FACULTYNAME"];
					member["full_name"] = gst_member["FACULTYNAME"];
					member["emplid"] = parseInt(gst_member["EMPLID_FAC"], 10);
					member["updated"] = gst_member["DATEUPDATED"];

					if (gst_member["ROLE"] == "C") {

						plan["chair"] = member;
						if (bc.isUndefinedOrIsNull(plan["professor_declared_date"])) {
							plan["professor_declared_date"] = gst_member["DATEUPDATED"];
						}
					}
					else if (gst_member["ROLE"] == "X") {

						if (bc.isUndefinedOrIsNull(plan["professor_declared_date"])) {
							plan["professor_declared_date"] = gst_member["DATEUPDATED"];
						}
						if (bc.isUndefinedOrIsNull(plan["co_chair1"])) {
							plan["co_chair1"] = member;
						}
						else {
							plan["co_chair2"] = member;
						}
					}
					else if (gst_member["ROLE"] == "O") {
						plan["univ_rep"] = member;
					}
					else if (gst_member["ROLE"] == "M") {
						if (bc.isUndefinedOrIsNull(plan["committee_established_date"])) {
							plan["committee_established_date"] = gst_member["DATEUPDATED"];
						}
						plan["members"].push(member);
					}
				} // next gst_member
			} // has some committee members
		} // found track
	} // next plan
} // update_student_from_gst_cmte

function reconcile_committees_with_faculty_data(students) {


//	var parse = require('csv-parse/lib/sync');

//	var faculty_csv = fs.readFileSync(faculty_path);
//	var faculty = parse(faculty_csv, {
//		'columns': true,
//		'objname': "fsuid"
//	});
	// console.log(faculty);
 
	for (var i = 0; i < students.length; i++) {
		var student = students[i];
		for (var plan_code in student.academic_plans) {
			var plan = student.academic_plans[plan_code];
			reconcile_plan_committee(plan, null);
		}
	}

} // reconcile_committees_with_faculty_data

function reconcile_plan_committee(plan, faculty) {

	if (bc.notUndefinedAndNotNull(plan["chair"])) {
		match_member_with_faculty(plan["chair"], faculty);
	}
	if (bc.notUndefinedAndNotNull(plan["co_chair1"])) {
		match_member_with_faculty(plan["co_chair1"], faculty);
	}
	if (bc.notUndefinedAndNotNull(plan["co_chair2"])) {
		match_member_with_faculty(plan["co_chair2"], faculty);
	}
	if (bc.notUndefinedAndNotNull(plan["univ_rep"])) {
		match_member_with_faculty(plan["univ_rep"], faculty);
	}
	if (bc.notUndefinedAndNotNull(plan["members"])) {
		// console.log(plan["members"]);
		for (var i = 0; i < plan["members"].length; i++) {

			match_member_with_faculty(plan["members"][i], faculty);
		}

		// plan["members"] = remove_duplicate_members(plan["members"]);
	}
}

function match_member_with_faculty(member, faculty) {

	if (member['dept'] && member['fsuid'] && member['email']) {

		return;
	}
	var foundProf = false;

//  for (var prof_key in faculty) {
//  		var prof = faculty[prof_key];
//  		//console.log(member["emplid"] + " == " + prof.emplid);
//  		if ((parseInt(member["emplid"], 10) == parseInt(prof.emplid, 10)) ||
//  			(member["fsuid"] == prof.fsuid)) {
//  			member['emplid'] = prof.emplid;
//  			member['dept'] = prof.dept;
//  			member['fsuid'] = prof.fsuid;
//  			member['email'] = prof.email;
//  			foundProf = true;
//  			break;
//  		}
//  
//  	}

	if (!foundProf && member['emplid']) {

		// try getting from ldap
		//  console.log(faculty);
		// console.log(member);
		var ldap_info = bc.getLDAPInfo(username, password, member["emplid"]);
		// console.log("ldapinfo: " + JSON.stringify(ldap_info, null, "	"));

		if (null != ldap_info.fsuid) {
			member['fsuid'] = ldap_info.fsuid;
		}
		else {
			member['fsuid'] = member['emplid'];
		}

		if (null != ldap_info.department) {
			member['dept'] = ldap_info.department;
		}
		if (null != ldap_info.email) {
			member['email'] = ldap_info.email;
		}
		if (null != ldap_info.givenName) {
			member['givenName'] = ldap_info.givenName;
		}
	} // looked up in ldap

} // match_member_with_faculty

function update_progression_from_gst_objects(students, gst_objects) {

	for (var i = 0; i < gst_objects.length; i++) {
		var gst_object = gst_objects[i];
		var student = match_student_with_gst_record_by_emplid(students, gst_object);
		if (null != student) {
			var plan_code = match_plan_with_gst_record_by_trackid(student, gst_object);
			if (null != plan_code) {
			update_student_from_gst_object(student, plan_code, gst_object);
			}
		}

	} // next gst_record

}

function update_student_from_gst_object(student, plan_code, gst_object) {
	/*
	{
		"STUDENTNAME": "Darrow, Emily M",
		"EMPLID": "000076998",
		"ACADPLANTEXT": "Biological Science - PHD",
		"ACADPROGTEXT": "DOCT",
		"STUDENTTRACKID": "82457",
		"STUDENTOBJECTID": "428934",
		"OBJECTSTATUS": "C",
		"STATUSDESCR": "Completed *",
		"OBJECTDATE": "",
		"COMMENTS": "",
		"SORTCODE": "1",
		"gst_id": "3",
		"plan_key": "program_of_studies_approved_date"
	}
	*/

	let plan = student['academic_plans'][plan_code];
	let plan_key = gst_object.plan_key;
	let value_type = gst_object.value_type;

	// TODO: how to handle "completed" objects that have no date?

	if ("" != gst_object["OBJECTDATE"]) {
		// TODO: make sure date is in expected format: MM/dd/YYYY
		plan[plan_key] = gst_object["OBJECTDATE"];
	}
	else if ("C" == gst_object["OBJECTSTATUS"] || "P" == gst_object["OBJECTSTATUS"] || "E" == gst_object["OBJECTSTATUS"]) {
		// don't overwrite if we already have it in local gradbeta database
		if (bc.isUndefinedOrIsNull(plan[plan_key])) {
			if ("date" == value_type) {
			plan[plan_key] = todays_date;
			}
			else {
			plan[plan_key] = gst_object["STATUSDESCR"];
			}
		}
	}

}

function update_scores_from_gst_scores(students, gst_scores) {

	for (var i = 0; i < gst_scores.length; i++) {
		var gst_score = gst_scores[i];
		var student = match_student_with_gst_record_by_emplid(students, gst_score);

		if (null != student) {
			update_student_from_gst_score(student, gst_score);
		}

	} // next gst_record

} // update_scores_from_gst_scores

function update_student_from_gst_score(student, gst_score) {

	if (gst_score["TEST_TYPE"] == "GRE") {
		if (gst_score["SUBJECT_TEST_DESC"] == "Verbal (Prior to 8/1/11)") {
			student["GRE_verbal"] = gst_score["TEST_SCORE"];
		}
		if (gst_score["SUBJECT_TEST_DESC"] == "Quantitative (Prior to 8/1/11)") {
			student["GRE_quantitative"] = gst_score["TEST_SCORE"];
		}
		if (gst_score["SUBJECT_TEST_DESC"] == "Verbal") {
			student["GRE_verbal"] = gst_score["TEST_SCORE"];
		}
		if (gst_score["SUBJECT_TEST_DESC"] == "Quantitative") {
			student["GRE_quantitative"] = gst_score["TEST_SCORE"];
		}
		if (gst_score["SUBJECT_TEST_DESC"] == "Writing") {
			student["GRE_writing"] = gst_score["TEST_SCORE"];
		}
	}

	if (gst_score["TEST_TYPE"] == "TOEFL") {
		if (gst_score["SUBJECT_TEST_DESC"] == "IBT Listening") {
			student["toefl_listen"] = gst_score["TEST_SCORE"];
		}
		if (gst_score["SUBJECT_TEST_DESC"] == "IBT Reading") {
			student["toefl_read"] = gst_score["TEST_SCORE"];
		}
		if (gst_score["SUBJECT_TEST_DESC"] == "IBT Speaking") {
			student["toefl_speak"] = gst_score["TEST_SCORE"];
		}
		if (gst_score["SUBJECT_TEST_DESC"] == "IBT Total Score") {
			student["toefl"] = gst_score["TEST_SCORE"];
		}
		if (gst_score["SUBJECT_TEST_DESC"] == "IBT Writing") {
			student["toefl_write"] = gst_score["TEST_SCORE"];
		}
	}

} // update_student_from_gst_score

function reset_admit_and_last_term(student) {

	var earliest_admit_months = 99999;
	var latest_last_months = -1;
	var has_active_plan = false;

	student["gst_track_id"] = [];
	student["all_tracks"] = {};
	
	for (let plan_code in student.academic_plans) {

		let plan = student.academic_plans[plan_code];
		// console.log(student.fsuid + " " + plan.admit_term);
		let admit_months = termcode2months(plan.admit_term);
		if (admit_months < earliest_admit_months) {
			earliest_admit_months = admit_months;
		}

		if (bc.notUndefinedAndNotNull(plan.last_term)) {
			let last_months = termcode2months(plan.last_term);
			if (last_months > latest_last_months) {
			latest_last_months = last_months;
			}
		}
		
		student["all_tracks"][plan["gst_trackid"]] = plan_code;

		if (plan.status == "Active in Program" && bc.isUndefinedOrIsNull(plan.last_term)) {
			has_active_plan = true;

			// TODO: allow for multiple plans with plan code...
			// if (bc.isUndefinedOrIsNull(student["academic_plan_code"])) {
			//				student["academic_plan_code"] = [];
			//	}
			// student["academic_plan_code"].push(plan_code);

			student["academic_plan_code"] = plan_code;
			student["gst_track_id"].push(plan["gst_trackid"]);

		}

		if (null != plan["discontinued"]) {
			student["discontinued"] = plan["discontinued"];
			// NOTE: does a student only have 1 final discontinued?
		}
		
		
	}
	if (latest_last_months > 0 && !has_active_plan) {
		student["last_term"] = Math.floor(latest_last_months / 12) + '/' + latest_last_months % 12;
	}
	else {
		student["last_term"] = null;
	}
	if (earliest_admit_months < 99999) {
		student["admit_term"] = Math.floor(earliest_admit_months / 12) + '/' + earliest_admit_months % 12;
	}

} // reset_admit_and_last_term

var allstudents;

if (updating) {



	allstudents = bc.readJSON(input_students_path);
	
	allstudents.sort(function(a,b) {

	if (a.fsuid < b.fsuid) {
		return -1;
	}
	if (a.fsuid > b.fsuid) {
		return 1;
	}
	return 0;
	
});

 fs.writeFileSync(pre_output_students_path, JSON.stringify(allstudents, null, '\t'));


}
else {
	allstudents = new Array;
}

update_students_from_gst_plans(allstudents,gst_plans);

update_scores_from_gst_scores(allstudents,gst_scores);

update_committees_from_gst_cmtes(allstudents,gst_cmtes);

reconcile_committees_with_faculty_data(allstudents);

update_progression_from_gst_objects(allstudents,gst_objects);

for (let i = 0; i< allstudents.length;i++) {
	reset_admit_and_last_term(allstudents[i]);
	bc.removeInvalidKeysFromObject(allstudents[i]);
}

// sort by fsuid
allstudents.sort(function(a,b) {

	if (a.fsuid < b.fsuid) {
		return -1;
	}
	if (a.fsuid > b.fsuid) {
		return 1;
	}
	return 0;
	
});

if (updating) {
	// delete mongo id to allow for uploading back into mongo
	allstudents.forEach(function(a) {

		if (bc.notUndefinedAndNotNull(a["_id"])) {
			delete a["_id"];
		}

	});

}


fs.writeFile(output_students_path,JSON.stringify(allstudents, null, '\t'),function(err) {
	if (err) {
		return console.log(err);
	}

}

); // fsWrite

console.log("\ngst-to-gradphile: found " + allstudents.length + " student records\n\n");