feat: started gradient descent

ml_code/optimization_functions.py

@@ -1,4 +1,37 @@
+import pytorch
+
 # Gradient Descent
+# https://machinelearningmastery.com/gradient-descent-optimization-from-scratch/
+# only works on differentiable functions
+# requires - target function & derivative of target function
+def gradient_descent(objective, derivative, bounds, n_iter, step_size):
+    # track all solutions
+    solutions, scores = list(), list()
+
+    # generate an initial point
+    solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
+
+    # run the gradient descent
+    for i in range(n_iter):
+
+        # calculate gradient
+        gradient = derivative(solution)
+
+        # take a step
+        solution = solution - step_size * gradient
+
+        # evaluate candidate point
+        solution_eval = objective(solution)
+
+        # store solution
+        solutions.append(solution)
+
+        scores.append(solution_eval)
+        # report progress
+        print('>%d f(%s) = %.5f' % (i, solution, solution_eval))
+
+    return [solutions, scores]
+
 
 # Stochastic Gradient Descent
 
@@ -14,4 +47,4 @@
 
 # Adam
 
-# Nadam
+# Nadam