Thinking process

instead of top down, this problem should be solved using bottom up

class ResultType {
    public TreeNode min;
    public TreeNode max;
    public int numOfNodes;

    public ResultType(TreeNode min, TreeNode max, int num) {
        this.min = min;
        this.max = max;
        this.numOfNodes = num;
    }
} 

public class Solution {
    private int largestBST;
    public int largestBSTSubtree(TreeNode root) {
        largestBST = 0;

        dfsHelper(root);

        return largestBST;
    }

    private ResultType dfsHelper(TreeNode root) {
        if (root == null) return new ResultType(null, null, 0);

        ResultType result = new ResultType(null, null, -1);
        ResultType left = dfsHelper(root.left);
        ResultType right = dfsHelper(root.right);

        if (left.numOfNodes == -1 || right.numOfNodes == -1) {
            return result;
        }

        if ((left.max == null || root.val > left.max.val || root.val == Integer.MAX_VALUE) && (right.min == null || root.val < right.min.val || root.val == Integer.MIN_VALUE)) {
            result.numOfNodes = left.numOfNodes + right.numOfNodes + 1; 
            largestBST = Math.max(result.numOfNodes, largestBST);

            result.max = right.max != null ? right.max : root;
            result.min = left.min != null ? left.min : root;
        }

        return result;
    }
}