Skip to content

Quantization

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
namespace LLama.Examples.Examples
{
    public class QuantizeModel
    {
        public static void Run()
        {
            string inputPath = UserSettings.GetModelPath();

            Console.Write("Please input your output model path: ");
            var outputPath = Console.ReadLine();

            Console.Write("Please input the quantize type (one of q4_0, q4_1, q5_0, q5_1, q8_0): ");
            var quantizeType = Console.ReadLine();

            if (LLamaQuantizer.Quantize(inputPath, outputPath, quantizeType))
            {
                Console.WriteLine("Quantization succeeded!");
            }
            else
            {
                Console.WriteLine("Quantization failed!");
            }
        }
    }
}