...
 
Commits (5)
......@@ -26,21 +26,9 @@
"* [How to Develop Convolutional Neural Network Models for Time Series Forecasting](https://machinelearningmastery.com/how-to-develop-convolutional-neural-network-models-for-time-series-forecasting/)\n",
"* [Introduction to Recurrent Networks in TensorFlow](https://danijar.com/introduction-to-recurrent-networks-in-tensorflow/)\n",
"* [Recurrent Neural Networks in Tensorflow I](https://r2rt.com/recurrent-neural-networks-in-tensorflow-i.html)\n",
"* [Recurrent Neural Networks in Tensorflow II](https://r2rt.com/recurrent-neural-networks-in-tensorflow-ii.html)\n",
"* [RNNs in Tensorflow, a Practical Guide and Undocumented Features](http://www.wildml.com/2016/08/rnns-in-tensorflow-a-practical-guide-and-undocumented-features/)\n",
"* [Time-Series Analysis Using Recurrent Neural Networks in Tensorflow](https://medium.com/themlblog/time-series-analysis-using-recurrent-neural-networks-in-tensorflow-2a0478b00be7)\n",
"* [Time Series Forecasting with the Long Short-Term Memory Network in Python](https://machinelearningmastery.com/time-series-forecasting-long-short-term-memory-network-python/)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Notes\n",
"\n",
"Investigate whether the neural networks should have a single input and output, multiple inputs and one output or multiple inputs and multiple outputs. Trying out multiple options isn't out of question either."
]
},
{
"cell_type": "markdown",
"metadata": {},
......@@ -298,6 +286,8 @@
" \n",
" \n",
"def create_placeholders(dataset):\n",
" \"\"\"Create input and target tensor placeholders.\"\"\"\n",
" \n",
" dtype = input_data[dataset]['training'].dtype\n",
" shape = (None, *input_data[dataset]['training'].shape[1:])\n",
" input_tensor = tensorflow.placeholder(dtype, shape)\n",
......@@ -310,6 +300,8 @@
"\n",
"\n",
"def create_feed_dicts(dataset, input_tensor, target_tensor):\n",
" \"\"\"Create training, validation and testing feed dictionaries.\"\"\"\n",
" \n",
" training_dict = {\n",
" input_tensor: input_data[dataset]['training'],\n",
" target_tensor: target_data[dataset]['training'],\n",
......@@ -329,6 +321,8 @@
"\n",
"\n",
"def train_rnn(dataset, rnn_graph_function, unit_count, layer_count, dropout_rate, learning_rate):\n",
" \"\"\"RNN training wrapper.\"\"\"\n",
" \n",
" tensorflow.reset_default_graph()\n",
" input_tensor, target_tensor = create_placeholders(dataset)\n",
" loss, optimisation_operation = rnn_graph_function(unit_count, layer_count, dropout_rate, learning_rate,\n",
......@@ -336,6 +330,7 @@
" \n",
" training_dict, validation_dict, testing_dict = create_feed_dicts(dataset, input_tensor, target_tensor)\n",
" \n",
" # Make sure dropout is active only during the training phase.\n",
" training_dict[tensorflow.keras.backend.learning_phase()] = 1\n",
" validation_dict[tensorflow.keras.backend.learning_phase()] = 0\n",
" testing_dict[tensorflow.keras.backend.learning_phase()] = 0\n",
......@@ -344,6 +339,8 @@
" \n",
" \n",
"def train_cnn(dataset, dropout_rate, learning_rate):\n",
" \"\"\"CNN training wrapper.\"\"\"\n",
" \n",
" tensorflow.reset_default_graph()\n",
" input_tensor, target_tensor = create_placeholders(dataset)\n",
" loss, optimisation_operation, is_training = cnn_graph(dropout_rate, learning_rate, input_tensor,\n",
......@@ -351,6 +348,7 @@
" \n",
" training_dict, validation_dict, testing_dict = create_feed_dicts(dataset, input_tensor, target_tensor)\n",
" \n",
" # Make sure dropout is active only during the training phase.\n",
" training_dict[is_training] = True\n",
" validation_dict[is_training] = False\n",
" testing_dict[is_training] = False\n",
......@@ -359,6 +357,8 @@
" \n",
" \n",
"def train_nn(loss, optimisation_operation, training_dict, validation_dict, testing_dict):\n",
" \"\"\"Train model and print results.\"\"\"\n",
" \n",
" lowest_validation_loss = float('inf')\n",
" stagnant_epoch_count = 0\n",
" \n",
......@@ -388,6 +388,9 @@
" print(s)\n",
" \n",
" model_manager.restore_model(session)\n",
" \n",
" print('lowest achieved validation loss:', lowest_validation_loss)\n",
" \n",
" testing_loss = session.run(loss, testing_dict)\n",
" print('loss on testing data:', testing_loss)"
]
......@@ -396,7 +399,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"## Naive Model Number One"
"## Naive Model Number One\n",
"\n",
"Outputs the last value of the input sequence."
]
},
{
......@@ -443,7 +448,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"## Naive Model Number Two"
"## Naive Model Number Two\n",
"\n",
"Outputs an average of all values of the input sequence."
]
},
{
......@@ -531,10 +538,26 @@
"metadata": {},
"outputs": [],
"source": [
"unit_count = 24\n",
"layer_count = 8\n",
"dropout_rate = 0.05\n",
"learning_rate = 0.001\n",
"hyperparameters = {\n",
" 'co2': {\n",
" 'unit_count': 16,\n",
" 'layer_count': 4,\n",
" 'dropout_rate': 0.1,\n",
" 'learning_rate': 0.005,\n",
" },\n",
" 'lake': {\n",
" 'unit_count': 16,\n",
" 'layer_count': 4,\n",
" 'dropout_rate': 0.1,\n",
" 'learning_rate': 0.005,\n",
" },\n",
" 'sunspot': {\n",
" 'unit_count': 6,\n",
" 'layer_count': 2,\n",
" 'dropout_rate': 0.05,\n",
" 'learning_rate': 0.002,\n",
" },\n",
"}\n",
"\n",
"is_first = True\n",
"for dataset in raw_data_paths.keys():\n",
......@@ -544,6 +567,11 @@
" print()\n",
" print('dataset:', dataset)\n",
" \n",
" unit_count = hyperparameters[dataset]['unit_count']\n",
" layer_count = hyperparameters[dataset]['layer_count']\n",
" dropout_rate = hyperparameters[dataset]['dropout_rate']\n",
" learning_rate = hyperparameters[dataset]['learning_rate']\n",
" \n",
" train_rnn(dataset, basic_rnn_graph, unit_count, layer_count, dropout_rate, learning_rate)"
]
},
......@@ -593,10 +621,26 @@
"metadata": {},
"outputs": [],
"source": [
"unit_count = 12\n",
"layer_count = 4\n",
"dropout_rate = 0.2\n",
"learning_rate = 0.001\n",
"hyperparameters = {\n",
" 'co2': {\n",
" 'unit_count': 12,\n",
" 'layer_count': 4,\n",
" 'dropout_rate': 0.4,\n",
" 'learning_rate': 0.01,\n",
" },\n",
" 'lake': {\n",
" 'unit_count': 12,\n",
" 'layer_count': 4,\n",
" 'dropout_rate': 0.4,\n",
" 'learning_rate': 0.01,\n",
" },\n",
" 'sunspot': {\n",
" 'unit_count': 8,\n",
" 'layer_count': 2,\n",
" 'dropout_rate': 0.05,\n",
" 'learning_rate': 0.002,\n",
" },\n",
"}\n",
"\n",
"is_first = True\n",
"for dataset in raw_data_paths.keys():\n",
......@@ -606,6 +650,11 @@
" print()\n",
" print('dataset:', dataset)\n",
" \n",
" unit_count = hyperparameters[dataset]['unit_count']\n",
" layer_count = hyperparameters[dataset]['layer_count']\n",
" dropout_rate = hyperparameters[dataset]['dropout_rate']\n",
" learning_rate = hyperparameters[dataset]['learning_rate']\n",
" \n",
" train_rnn(dataset, lstm_graph, unit_count, layer_count, dropout_rate, learning_rate)"
]
},
......@@ -655,10 +704,26 @@
"metadata": {},
"outputs": [],
"source": [
"unit_count = 12\n",
"layer_count = 4\n",
"dropout_rate = 0.2\n",
"learning_rate = 0.001\n",
"hyperparameters = {\n",
" 'co2': {\n",
" 'unit_count': 12,\n",
" 'layer_count': 4,\n",
" 'dropout_rate': 0.4,\n",
" 'learning_rate': 0.01,\n",
" },\n",
" 'lake': {\n",
" 'unit_count': 12,\n",
" 'layer_count': 4,\n",
" 'dropout_rate': 0.4,\n",
" 'learning_rate': 0.01,\n",
" },\n",
" 'sunspot': {\n",
" 'unit_count': 8,\n",
" 'layer_count': 2,\n",
" 'dropout_rate': 0.05,\n",
" 'learning_rate': 0.002,\n",
" },\n",
"}\n",
"\n",
"is_first = True\n",
"for dataset in raw_data_paths.keys():\n",
......@@ -668,6 +733,11 @@
" print()\n",
" print('dataset:', dataset)\n",
" \n",
" unit_count = hyperparameters[dataset]['unit_count']\n",
" layer_count = hyperparameters[dataset]['layer_count']\n",
" dropout_rate = hyperparameters[dataset]['dropout_rate']\n",
" learning_rate = hyperparameters[dataset]['learning_rate']\n",
" \n",
" train_rnn(dataset, gru_graph, unit_count, layer_count, dropout_rate, learning_rate)"
]
},
......