/*
 * Copyright (c) 2014, Shogun Toolbox Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:

 * 1. Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.

 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * Written (W) 2014 Khaled Nasr
 */

#include <shogun/neuralnets/RBM.h>


#include <shogun/base/Parameter.h>
#include <shogun/mathematics/Math.h>
#include <shogun/mathematics/eigen3.h>

using namespace shogun;

CRBM::CRBM() : CSGObject()
{
	init();
}

CRBM::CRBM(int32_t num_hidden)
{
	init();
	m_num_hidden = num_hidden;
}

CRBM::CRBM(int32_t num_hidden, int32_t num_visible,
	ERBMVisibleUnitType visible_unit_type) : CSGObject()
{
	init();
	m_num_hidden = num_hidden;
	add_visible_group(num_visible, visible_unit_type);
}

CRBM::~CRBM()
{
	SG_UNREF(m_visible_group_sizes);
	SG_UNREF(m_visible_group_types);
	SG_UNREF(m_visible_state_offsets);
}

void CRBM::add_visible_group(int32_t num_units, ERBMVisibleUnitType unit_type)
{
	m_num_visible_groups++;
	m_num_visible += num_units;

	m_visible_group_sizes->append_element(num_units);
	m_visible_group_types->append_element(unit_type);

	int32_t n = m_visible_state_offsets->get_num_elements();

	if (n==0)
		m_visible_state_offsets->append_element(0);
	else
		m_visible_state_offsets->append_element(
			m_visible_state_offsets->element(n-1)+m_visible_group_sizes->element(n-1));
}

void CRBM::initialize_neural_network(float64_t sigma)
{
	m_num_params = m_num_visible + m_num_hidden + m_num_visible*m_num_hidden;
	m_params = SGVector<float64_t>(m_num_params);

	for (int32_t i=0; i<m_num_params; i++)
		m_params[i] = CMath::normal_random(0.0,sigma);
}

void CRBM::set_batch_size(int32_t batch_size)
{
	if (m_batch_size == batch_size) return;

	m_batch_size = batch_size;

	hidden_state = SGMatrix<float64_t>(m_num_hidden,m_batch_size);
	visible_state = SGMatrix<float64_t>(m_num_visible,m_batch_size);

	reset_chain();
}

void CRBM::train(CDenseFeatures<float64_t>* features)
{
	REQUIRE(features != NULL, "Invalid (NULL) feature pointer\n");
	REQUIRE(features->get_num_features()==m_num_visible,
		"Number of features (%i) must match the RBM's number of visible units "
		"(%i)\n", features->get_num_features(), m_num_visible);

	SGMatrix<float64_t> inputs = features->get_feature_matrix();

	int32_t training_set_size = inputs.num_cols;
	if (gd_mini_batch_size==0) gd_mini_batch_size = training_set_size;
	set_batch_size(gd_mini_batch_size);

	for (int32_t i=0; i<m_num_visible; i++)
		for (int32_t j=0; j<m_batch_size; j++)
			visible_state(i,j) = inputs(i,j);

	SGVector<float64_t> gradients(m_num_params);

	// needed for momentum
	SGVector<float64_t> param_updates(m_num_params);
	param_updates.zero();

	float64_t alpha = gd_learning_rate;

	SGMatrix<float64_t> buffer;
	if (monitoring_method == RBMMM_RECONSTRUCTION_ERROR)
		buffer = SGMatrix<float64_t>(m_num_visible, m_batch_size);
	else if (monitoring_method == RBMMM_PSEUDO_LIKELIHOOD)
		buffer = SGMatrix<float64_t>(m_num_hidden, m_batch_size);

	int32_t counter = 0;
	for (int32_t i=0; i<max_num_epochs; i++)
	{
		for (int32_t j=0; j < training_set_size; j += gd_mini_batch_size)
		{
			alpha = gd_learning_rate_decay*alpha;

			if (j+gd_mini_batch_size>training_set_size)
				j = training_set_size-gd_mini_batch_size;

			SGMatrix<float64_t> inputs_batch(inputs.matrix+j*inputs.num_rows,
				inputs.num_rows, gd_mini_batch_size, false);

			for (int32_t k=0; k<m_num_params; k++)
				m_params[k] += gd_momentum*param_updates[k];

			contrastive_divergence(inputs_batch, gradients);

			for (int32_t k=0; k<m_num_params; k++)
			{
				param_updates[k] = gd_momentum*param_updates[k]
						-alpha*gradients[k];

				m_params[k] -= alpha*gradients[k];
			}

			if (counter%monitoring_interval == 0)
			{
				if (monitoring_method==RBMMM_RECONSTRUCTION_ERROR)
					SG_INFO("Epoch %i: reconstruction Error = %f\n",i,
						reconstruction_error(inputs_batch, buffer));
				if (monitoring_method==RBMMM_PSEUDO_LIKELIHOOD)
					SG_INFO("Epoch %i: Pseudo-log-likelihood = %f\n",i,
						pseudo_likelihood(inputs_batch,buffer));
			}
			counter ++;
		}
	}
}

void CRBM::sample(int32_t num_gibbs_steps,
	int32_t batch_size)
{
	set_batch_size(batch_size);

	for (int32_t i=0; i<num_gibbs_steps; i++)
	{
		mean_hidden(visible_state, hidden_state);
		sample_hidden(hidden_state, hidden_state);
		mean_visible(hidden_state, visible_state);
		if (i<num_gibbs_steps-1)
			sample_visible(visible_state, visible_state);
	}
}

CDenseFeatures< float64_t >* CRBM::sample_group(int32_t V,
	int32_t num_gibbs_steps, int32_t batch_size)
{
	REQUIRE(V<m_num_visible_groups,
		"Visible group index (%i) out of bounds (%i)\n", V, m_num_visible);

	sample(num_gibbs_steps, batch_size);

	SGMatrix<float64_t> result(m_visible_group_sizes->element(V), m_batch_size);

	int32_t offset = m_visible_state_offsets->element(V);
	for (int32_t i=0; i<m_visible_group_sizes->element(V); i++)
		for (int32_t j=0; j<m_batch_size; j++)
			result(i,j) = visible_state(i+offset,j);

	return new CDenseFeatures<float64_t>(result);
}

void CRBM::sample_with_evidence(
	int32_t E, CDenseFeatures< float64_t >* evidence, int32_t num_gibbs_steps)
{
	REQUIRE(E<m_num_visible_groups,
		"Visible group index (%i) out of bounds (%i)\n", E, m_num_visible);

	set_batch_size(evidence->get_num_vectors());

	SGMatrix<float64_t> evidence_matrix = evidence->get_feature_matrix();

	int32_t offset = m_visible_state_offsets->element(E);

	for (int32_t i=0; i<m_visible_group_sizes->element(E); i++)
		for (int32_t j=0; j<m_batch_size; j++)
			visible_state(i+offset,j) = evidence_matrix(i,j);

	for (int32_t n=0; n<num_gibbs_steps; n++)
	{
		mean_hidden(visible_state, hidden_state);
		sample_hidden(hidden_state, hidden_state);
		mean_visible(hidden_state, visible_state);
		if (n<num_gibbs_steps-1)
		{
			for (int32_t k=0; k<m_num_visible_groups; k++)
				if (k!=E)
					sample_visible(k, visible_state, visible_state);
		}

		for (int32_t i=0; i<m_visible_group_sizes->element(E); i++)
			for (int32_t j=0; j<m_batch_size; j++)
				visible_state(i+offset,j) = evidence_matrix(i,j);
	}
}

CDenseFeatures< float64_t >* CRBM::sample_group_with_evidence(int32_t V,
	int32_t E, CDenseFeatures< float64_t >* evidence, int32_t num_gibbs_steps)
{
	REQUIRE(V<m_num_visible_groups,
		"Visible group index (%i) out of bounds (%i)\n", V, m_num_visible);
	REQUIRE(E<m_num_visible_groups,
		"Visible group index (%i) out of bounds (%i)\n", E, m_num_visible);

	sample_with_evidence(E, evidence, num_gibbs_steps);

	SGMatrix<float64_t> result(m_visible_group_sizes->element(V), m_batch_size);

	int32_t offset = m_visible_state_offsets->element(V);
	for (int32_t i=0; i<m_visible_group_sizes->element(V); i++)
		for (int32_t j=0; j<m_batch_size; j++)
			result(i,j) = visible_state(i+offset,j);

	return new CDenseFeatures<float64_t>(result);
}

void CRBM::reset_chain()
{
	for (int32_t i=0; i<m_num_visible; i++)
		for (int32_t j=0; j<m_batch_size; j++)
			visible_state(i,j) = CMath::random(0.0,1.0) > 0.5;
}

float64_t CRBM::free_energy(SGMatrix< float64_t > visible, SGMatrix< float64_t > buffer)
{
	set_batch_size(visible.num_cols);

	if (buffer.num_rows==0)
		buffer = SGMatrix<float64_t>(m_num_hidden, m_batch_size);

	typedef Eigen::Map<Eigen::MatrixXd> EMatrix;
	typedef Eigen::Map<Eigen::VectorXd> EVector;

	EMatrix V(visible.matrix, visible.num_rows, visible.num_cols);
	EMatrix W(get_weights().matrix, m_num_hidden, m_num_visible);
	EVector B(get_visible_bias().vector, m_num_visible);
	EVector C(get_hidden_bias().vector, m_num_hidden);

	EVector bv_buffer(buffer.matrix, m_batch_size);
	EMatrix wv_buffer(buffer.matrix, m_num_hidden, m_batch_size);

	bv_buffer = B.transpose()*V;
	float64_t bv_term = bv_buffer.sum();

	wv_buffer.colwise() = C;
	wv_buffer += W*V;

	float64_t wv_term = 0;
	for (int32_t i=0; i<m_num_hidden; i++)
		for (int32_t j=0; j<m_batch_size; j++)
			wv_term += CMath::log(1.0+CMath::exp(wv_buffer(i,j)));

	float64_t F = -1.0*(bv_term+wv_term)/m_batch_size;

	for (int32_t k=0; k<m_num_visible_groups; k++)
	{
		if (m_visible_group_types->element(k) == RBMVUT_GAUSSIAN)
		{
			int32_t offset = m_visible_state_offsets->element(k);

			for (int32_t i=0; i<m_visible_group_sizes->element(k); i++)
				for (int32_t j=0; j<m_batch_size; j++)
					F += 0.5*CMath::pow(visible(i+offset,j),2)/m_batch_size;
		}
	}

	return F;
}

void CRBM::free_energy_gradients(SGMatrix< float64_t > visible,
	SGVector< float64_t > gradients,
	bool positive_phase,
	SGMatrix< float64_t > hidden_mean_given_visible)
{
	set_batch_size(visible.num_cols);

	if (hidden_mean_given_visible.num_rows==0)
	{
		hidden_mean_given_visible = SGMatrix<float64_t>(m_num_hidden,m_batch_size);
		mean_hidden(visible, hidden_mean_given_visible);
	}

	typedef Eigen::Map<Eigen::MatrixXd> EMatrix;
	typedef Eigen::Map<Eigen::VectorXd> EVector;

	EMatrix V(visible.matrix, visible.num_rows, visible.num_cols);
	EMatrix PH(hidden_mean_given_visible.matrix, m_num_hidden,m_batch_size);

	EMatrix WG(get_weights(gradients).matrix, m_num_hidden, m_num_visible);
	EVector BG(get_visible_bias(gradients).vector, m_num_visible);
	EVector CG(get_hidden_bias(gradients).vector, m_num_hidden);

	if (positive_phase)
	{
		WG = -1*PH*V.transpose()/m_batch_size;
		BG = -1*V.rowwise().sum()/m_batch_size;
		CG = -1*PH.rowwise().sum()/m_batch_size;
	}
	else
	{
		WG += PH*V.transpose()/m_batch_size;
		BG += V.rowwise().sum()/m_batch_size;
		CG += PH.rowwise().sum()/m_batch_size;
	}
}

void CRBM::contrastive_divergence(SGMatrix< float64_t > visible_batch,
	SGVector< float64_t > gradients)
{
	set_batch_size(visible_batch.num_cols);

	// positive phase
	mean_hidden(visible_batch, hidden_state);
	free_energy_gradients(visible_batch, gradients, true, hidden_state);

	// sampling
	for (int32_t i=0; i<cd_num_steps; i++)
	{
		if (i>0 || cd_persistent)
			mean_hidden(visible_state, hidden_state);
		sample_hidden(hidden_state, hidden_state);
		mean_visible(hidden_state, visible_state);
		if (cd_sample_visible)
			sample_visible(visible_state, visible_state);
	}

	// negative phase
	mean_hidden(visible_state, hidden_state);
	free_energy_gradients(visible_state, gradients, false, hidden_state);

	// regularization
	if (l2_coefficient>0)
	{
		int32_t len = m_num_hidden*m_num_visible;
		for (int32_t i=0; i<len; i++)
			gradients[i+m_num_visible+m_num_hidden] +=
				l2_coefficient * m_params[i+m_num_visible+m_num_hidden];
	}

	if (l1_coefficient>0)
	{
		int32_t len = m_num_hidden*m_num_visible;
		for (int32_t i=0; i<len; i++)
			gradients[i+m_num_visible+m_num_hidden] +=
				l1_coefficient * m_params[i+m_num_visible+m_num_hidden];
	}

}

float64_t CRBM::reconstruction_error(SGMatrix< float64_t > visible,
	SGMatrix< float64_t > buffer)
{
	set_batch_size(visible.num_cols);

	if (buffer.num_rows==0)
		buffer = SGMatrix<float64_t>(m_num_visible, m_batch_size);

	mean_hidden(visible, hidden_state);
	sample_hidden(hidden_state, hidden_state);
	mean_visible(hidden_state, buffer);

	float64_t error = 0;

	int32_t len = m_num_visible*m_batch_size;
	for (int32_t i=0; i<len; i++)
			error += CMath::pow(buffer[i]-visible[i],2);

	return error/m_batch_size;
}


float64_t CRBM::pseudo_likelihood(SGMatrix< float64_t > visible,
	SGMatrix< float64_t > buffer)
{
	for (int32_t k=0; k<m_num_visible_groups; k++)
		if (m_visible_group_types->element(k)!=RBMVUT_BINARY)
			SG_ERROR("Pseudo-likelihood is only supported for binary visible units\n");

	set_batch_size(visible.num_cols);

	if (buffer.num_rows==0)
	buffer = SGMatrix<float64_t>(m_num_hidden, m_batch_size);

	SGVector<int32_t> indices(m_batch_size);
	for (int32_t i=0; i<m_batch_size; i++)
		indices[i] = CMath::random(0,m_num_visible-1);


	float64_t f1 = free_energy(visible, buffer);

	for (int32_t j=0; j<m_batch_size; j++)
		visible(indices[j],j) = 1.0-visible(indices[j],j);

	float64_t f2 = free_energy(visible, buffer);

	for (int32_t j=0; j<m_batch_size; j++)
		visible(indices[j],j) = 1.0-visible(indices[j],j);

	return m_num_visible*CMath::log(1.0/(1+CMath::exp(f1-f2)));
}

void CRBM::mean_hidden(SGMatrix< float64_t > visible, SGMatrix< float64_t > result)
{
	typedef Eigen::Map<Eigen::MatrixXd> EMatrix;
	typedef Eigen::Map<Eigen::VectorXd> EVector;

	EMatrix V(visible.matrix, visible.num_rows, visible.num_cols);
	EMatrix H(result.matrix, result.num_rows, result.num_cols);
	EMatrix W(get_weights().matrix, m_num_hidden, m_num_visible);
	EVector C(get_hidden_bias().vector, m_num_hidden);

	H.colwise() = C;
	H += W*V;

	int32_t len = result.num_rows*result.num_cols;
	for (int32_t i=0; i<len; i++)
		result[i] = 1.0/(1.0+CMath::exp(-1.0*result[i]));
}

void CRBM::mean_visible(SGMatrix< float64_t > hidden, SGMatrix< float64_t > result)
{
	typedef Eigen::Map<Eigen::MatrixXd> EMatrix;
	typedef Eigen::Map<Eigen::VectorXd> EVector;

	EMatrix H(hidden.matrix, hidden.num_rows, hidden.num_cols);
	EMatrix V(result.matrix, result.num_rows, result.num_cols);
	EMatrix W(get_weights().matrix, m_num_hidden, m_num_visible);
	EVector B(get_visible_bias().vector, m_num_visible);

	V.colwise() = B;
	V += W.transpose()*H;

	for (int32_t k=0; k<m_num_visible_groups; k++)
	{
		int32_t offset = m_visible_state_offsets->element(k);

		if (m_visible_group_types->element(k)==RBMVUT_BINARY)
		{
			for (int32_t i=0; i<m_visible_group_sizes->element(k); i++)
				for (int32_t j=0; j<m_batch_size; j++)
					result(i+offset,j) = 1.0/(1.0+CMath::exp(-1.0*result(i+offset,j)));
		}
		if (m_visible_group_types->element(k)==RBMVUT_SOFTMAX)
		{
			// to avoid exponentiating large numbers, the maximum activation is
			// subtracted from all the activations and the computations are done
			// in thelog domain

			float64_t max = result(offset,0);
			for (int32_t i=0; i<m_visible_group_sizes->element(k); i++)
				for (int32_t j=0; j<m_batch_size; j++)
					if (result(i+offset,j) > max)
						max = result(i+offset,j);

			for (int32_t j=0; j<m_batch_size; j++)
			{
				float64_t sum = 0;
				for (int32_t i=0; i<m_visible_group_sizes->element(k); i++)
					sum += CMath::exp(result(i+offset,j)-max);

				float64_t normalizer = CMath::log(sum);

				for (int32_t i=0; i<m_visible_group_sizes->element(k); i++)
					result(i+offset,j) =
						CMath::exp(result(i+offset,j)-max-normalizer);
			}
		}
	}
}

void CRBM::sample_hidden(SGMatrix< float64_t > mean, SGMatrix< float64_t > result)
{
	int32_t length = result.num_rows*result.num_cols;
	for (int32_t i=0; i<length; i++)
		result[i] = CMath::random(0.0,1.0) < mean[i];
}

void CRBM::sample_visible(SGMatrix< float64_t > mean, SGMatrix< float64_t > result)
{
	for (int32_t k=0; k<m_num_visible_groups; k++)
	{
		sample_visible(k, mean, result);
	}
}

void CRBM::sample_visible(int32_t index,
	SGMatrix< float64_t > mean, SGMatrix< float64_t > result)
{
	int32_t offset = m_visible_state_offsets->element(index);

	if (m_visible_group_types->element(index)==RBMVUT_BINARY)
	{
		for (int32_t i=0; i<m_visible_group_sizes->element(index); i++)
			for (int32_t j=0; j<m_batch_size; j++)
				result(i+offset,j) = CMath::random(0.0,1.0) < mean(i+offset,j);
	}

	if (m_visible_group_types->element(index)==RBMVUT_SOFTMAX)
	{
		for (int32_t i=0; i<m_visible_group_sizes->element(index); i++)
			for (int32_t j=0; j<m_batch_size; j++)
				result(i+offset,j) = 0;

		for (int32_t j=0; j<m_batch_size; j++)
		{
			int32_t r = CMath::random(0.0,1.0);
			float64_t sum = 0;
			for (int32_t i=0; i<m_visible_group_sizes->element(index); i++)
			{
				sum += mean(i+offset,j);
				if (r<=sum)
				{
					result(i+offset,j) = 1;
					break;
				}
			}
		}
	}
}


SGMatrix< float64_t > CRBM::get_weights(SGVector< float64_t > p)
{
	if (p.vlen==0)
		return SGMatrix<float64_t>(m_params.vector+m_num_visible,
			m_num_hidden, m_num_visible, false);
	else
		return SGMatrix<float64_t>(p.vector+m_num_visible,
			m_num_hidden, m_num_visible, false);
}

SGVector< float64_t > CRBM::get_hidden_bias(SGVector< float64_t > p)
{
	if (p.vlen==0)
		return SGVector<float64_t>(m_params.vector+m_num_visible+m_num_visible*m_num_hidden,
			m_num_hidden, false);
	else
		return SGVector<float64_t>(p.vector+m_num_visible+m_num_visible*m_num_hidden,
			m_num_hidden, false);
}

SGVector< float64_t > CRBM::get_visible_bias(SGVector< float64_t > p)
{
	if (p.vlen==0)
		return SGVector<float64_t>(m_params.vector, m_num_visible, false);
	else
		return SGVector<float64_t>(p.vector, m_num_visible, false);
}

void CRBM::init()
{
	cd_num_steps = 1;
	cd_persistent = true;
	cd_sample_visible = false;
	l2_coefficient = 0.0;
	l1_coefficient = 0.0;
	monitoring_method = RBMMM_RECONSTRUCTION_ERROR;
	monitoring_interval = 10;

	gd_mini_batch_size = 0;
	max_num_epochs = 1;
	gd_learning_rate = 0.1;
	gd_learning_rate_decay = 1.0;
	gd_momentum = 0.9;

	m_num_hidden = 0;
	m_num_visible = 0;
	m_num_visible_groups = 0;
	m_visible_group_sizes = new CDynamicArray<int32_t>();
	m_visible_group_types = new CDynamicArray<int32_t>();
	m_visible_state_offsets = new CDynamicArray<int32_t>();
	m_num_params = 0;
	m_batch_size = 0;

	SG_ADD(&cd_num_steps, "cd_num_steps", "Number of CD Steps", MS_NOT_AVAILABLE);
	SG_ADD(&cd_persistent, "cd_persistent", "Whether to use PCD", MS_NOT_AVAILABLE);
	SG_ADD(&cd_sample_visible, "sample_visible",
		"Whether to sample the visible units during (P)CD", MS_NOT_AVAILABLE);
	SG_ADD(&l2_coefficient, "l2_coefficient",
	       "L2 regularization coeff", MS_NOT_AVAILABLE);
	SG_ADD(&l1_coefficient, "l1_coefficient",
	       "L1 regularization coeff", MS_NOT_AVAILABLE);
	SG_ADD((machine_int_t*)&monitoring_method, "monitoring_method",
		"Monitoring Method", MS_NOT_AVAILABLE);
	SG_ADD(&monitoring_interval, "monitoring_interval",
		"Monitoring Interval", MS_NOT_AVAILABLE);

	SG_ADD(&gd_mini_batch_size, "gd_mini_batch_size",
	       "Gradient Descent Mini-batch size", MS_NOT_AVAILABLE);
	SG_ADD(&max_num_epochs, "max_num_epochs",
	       "Max number of Epochs", MS_NOT_AVAILABLE);
	SG_ADD(&gd_learning_rate, "gd_learning_rate",
	       "Gradient descent learning rate", MS_NOT_AVAILABLE);
	SG_ADD(&gd_learning_rate_decay, "gd_learning_rate_decay",
	       "Gradient descent learning rate decay", MS_NOT_AVAILABLE);
	SG_ADD(&gd_momentum, "gd_momentum",
	       "Gradient Descent Momentum", MS_NOT_AVAILABLE);

	SG_ADD(&m_num_hidden, "num_hidden",
	       "Number of Hidden Units", MS_NOT_AVAILABLE);
	SG_ADD(&m_num_visible, "num_visible",
	       "Number of Visible Units", MS_NOT_AVAILABLE);

	SG_ADD(&m_num_visible_groups, "num_visible_groups",
	       "Number of Visible Unit Groups", MS_NOT_AVAILABLE);
	SG_ADD((CSGObject**)&m_visible_group_sizes, "visible_group_sizes",
	       "Sizes of Visible Unit Groups", MS_NOT_AVAILABLE);
	SG_ADD((CSGObject**)&m_visible_group_types, "visible_group_types",
	       "Types of Visible Unit Groups", MS_NOT_AVAILABLE);
	SG_ADD((CSGObject**)&m_visible_state_offsets, "visible_group_index_offsets",
	       "State Index offsets of Visible Unit Groups", MS_NOT_AVAILABLE);

	SG_ADD(&m_num_params, "num_params",
	       "Number of Parameters", MS_NOT_AVAILABLE);
	SG_ADD(&m_params, "params", "Parameters", MS_NOT_AVAILABLE);
}

